org.apache.flink.runtime.event.AbstractEvent Java Examples

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

private void processBufferOrEvent(BufferOrEvent bufferOrEvent) throws IOException {
	if (bufferOrEvent.isBuffer()) {
		lastChannel = channelIndexes.get(bufferOrEvent.getChannelInfo());
		checkState(lastChannel != StreamTaskInput.UNSPECIFIED);
		currentRecordDeserializer = recordDeserializers[lastChannel];
		checkState(currentRecordDeserializer != null,
			"currentRecordDeserializer has already been released");

		currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
	}
	else {
		// Event received
		final AbstractEvent event = bufferOrEvent.getEvent();
		// TODO: with checkpointedInputGate.isFinished() we might not need to support any events on this level.
		if (event.getClass() != EndOfPartitionEvent.class) {
			throw new IOException("Unexpected event: " + event);
		}

		// release the record deserializer immediately,
		// which is very valuable in case of bounded stream
		releaseDeserializer(channelIndexes.get(bufferOrEvent.getChannelInfo()));
	}
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

static void assertNextEvent(
		ResultSubpartitionView readView,
		int expectedReadableBufferSize,
		Class<? extends AbstractEvent> expectedEventClass,
		boolean expectedIsMoreAvailable,
		int expectedBuffersInBacklog,
		boolean expectedNextBufferIsEvent,
		boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException {
	assertNextBufferOrEvent(
		readView,
		expectedReadableBufferSize,
		false,
		expectedEventClass,
		expectedIsMoreAvailable,
		expectedBuffersInBacklog,
		expectedNextBufferIsEvent,
		expectedRecycledAfterRecycle);
}
 

@Override
protected void deserializeBuffer(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		deserializer.setNextBuffer(buffer);

		while (deserializer.hasUnfinishedData()) {
			RecordDeserializer.DeserializationResult result =
				deserializer.getNextRecord(delegate);

			if (result.isFullRecord()) {
				output.add(delegate.getInstance());
			}

			if (result == RecordDeserializer.DeserializationResult.LAST_RECORD_FROM_BUFFER
				|| result == RecordDeserializer.DeserializationResult.PARTIAL_RECORD) {
				break;
			}
		}
	} else {
		// is event
		AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		output.add(event);
	}
}
 

@Test
public void testToBuffer() throws IOException {
	for (AbstractEvent evt : events) {
		Buffer buffer = EventSerializer.toBuffer(evt);

		assertFalse(buffer.isBuffer());
		assertTrue(buffer.readableBytes() > 0);
		assertFalse(buffer.isRecycled());

		if (evt instanceof CheckpointBarrier) {
			assertTrue(buffer.getDataType().isBlockingUpstream());
		} else {
			assertEquals(Buffer.DataType.EVENT_BUFFER, buffer.getDataType());
		}
	}
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	AbstractEvent[] events = {
			EndOfPartitionEvent.INSTANCE,
			EndOfSuperstepEvent.INSTANCE,
			new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
			new TestTaskEvent(Math.random(), 12361231273L),
			new CancelCheckpointMarker(287087987329842L)
	};

	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}
 

@Test
public void testToBufferConsumer() throws IOException {
	for (AbstractEvent evt : events) {
		BufferConsumer bufferConsumer = EventSerializer.toBufferConsumer(evt);

		assertFalse(bufferConsumer.isBuffer());
		assertTrue(bufferConsumer.isFinished());
		assertTrue(bufferConsumer.isDataAvailable());
		assertFalse(bufferConsumer.isRecycled());

		if (evt instanceof CheckpointBarrier) {
			assertTrue(bufferConsumer.build().getDataType().isBlockingUpstream());
		} else {
			assertEquals(Buffer.DataType.EVENT_BUFFER, bufferConsumer.build().getDataType());
		}
	}
}
 

private BufferOrEvent transformToBufferOrEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, currentChannel.getChannelIndex(), moreAvailable);
	}
	else {
		final AbstractEvent event;
		try {
			event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
		}
		finally {
			buffer.recycleBuffer();
		}

		if (event.getClass() == EndOfPartitionEvent.class) {
			channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

			if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
				// Because of race condition between:
				// 1. releasing inputChannelsWithData lock in this method and reaching this place
				// 2. empty data notification that re-enqueues a channel
				// we can end up with moreAvailable flag set to true, while we expect no more data.
				checkState(!moreAvailable || !pollNext().isPresent());
				moreAvailable = false;
				hasReceivedAllEndOfPartitionEvents = true;
				markAvailable();
			}

			currentChannel.notifySubpartitionConsumed();
			currentChannel.releaseAllResources();
		}

		return new BufferOrEvent(event, currentChannel.getChannelIndex(), moreAvailable, buffer.getSize());
	}
}
 

public BufferOrEvent(AbstractEvent event, int channelIndex, boolean moreAvailable, int size) {
	this.buffer = null;
	this.event = checkNotNull(event);
	this.channelIndex = channelIndex;
	this.moreAvailable = moreAvailable;
	this.size = size;
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

/**
 * Returns the result of
 * {@link EventSerializer#isEvent(Buffer, Class)} on a buffer
 * that encodes the given <tt>event</tt>.
 *
 * @param event the event to encode
 * @param eventClass the event class to check against
 *
 * @return whether {@link EventSerializer#isEvent(ByteBuffer, Class)}
 * 		thinks the encoded buffer matches the class
 */
private boolean checkIsEvent(
		AbstractEvent event,
		Class<?> eventClass) throws IOException {

	final Buffer serializedEvent = EventSerializer.toBuffer(event);
	try {
		return EventSerializer.isEvent(serializedEvent, eventClass);
	} finally {
		serializedEvent.recycleBuffer();
	}
}
 

/**
 * Handles the event and returns whether the reader reached an end-of-stream event (either the
 * end of the whole stream or the end of an superstep).
 */
protected boolean handleEvent(AbstractEvent event) throws IOException {
	final Class<?> eventType = event.getClass();

	try {
		// ------------------------------------------------------------
		// Runtime events
		// ------------------------------------------------------------

		// This event is also checked at the (single) input gate to release the respective
		// channel, at which it was received.
		if (eventType == EndOfPartitionEvent.class) {
			return true;
		}
		else if (eventType == EndOfSuperstepEvent.class) {
			return incrementEndOfSuperstepEventAndCheck();
		}

		// ------------------------------------------------------------
		// Task events (user)
		// ------------------------------------------------------------
		else if (event instanceof TaskEvent) {
			taskEventHandler.publish((TaskEvent) event);

			return false;
		}
		else {
			throw new IllegalStateException("Received unexpected event of type " + eventType + " at reader.");
		}
	}
	catch (Throwable t) {
		throw new IOException("Error while handling event of type " + eventType + ": " + t.getMessage(), t);
	}
}
 

public void broadcastEvent(AbstractEvent event) throws IOException {
	try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event)) {
		for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) {
			tryFinishCurrentBufferBuilder(targetChannel);

			// Retain the buffer so that it can be recycled by each channel of targetPartition
			targetPartition.addBufferConsumer(eventBufferConsumer.copy(), targetChannel);
		}

		if (flushAlways) {
			flushAll();
		}
	}
}
 

public void sendEvent(AbstractEvent event, int channel) {
	synchronized (inputQueues[channel]) {
		inputQueues[channel].add(InputValue.event(event));
		inputQueues[channel].notifyAll();
	}
	inputGate.notifyChannelNonEmpty(inputChannels[channel]);
}
 

public void broadcastEvent(AbstractEvent event) throws IOException {
	try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event)) {
		for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) {
			tryFinishCurrentBufferBuilder(targetChannel);

			// Retain the buffer so that it can be recycled by each channel of targetPartition
			targetPartition.addBufferConsumer(eventBufferConsumer.copy(), targetChannel);
		}

		if (flushAlways) {
			flushAll();
		}
	}
}
 

/**
 * Tests {@link EventSerializer#isEvent(Buffer, Class)} returns
 * the correct answer for various encoded event buffers.
 */
@Test
public void testIsEvent() throws Exception {
	AbstractEvent[] events = {
		EndOfPartitionEvent.INSTANCE,
		EndOfSuperstepEvent.INSTANCE,
		new CheckpointBarrier(1678L, 4623784L, CheckpointOptions.forCheckpointWithDefaultLocation()),
		new TestTaskEvent(Math.random(), 12361231273L),
		new CancelCheckpointMarker(287087987329842L)
	};

	Class[] expectedClasses = Arrays.stream(events)
		.map(AbstractEvent::getClass)
		.toArray(Class[]::new);

	for (AbstractEvent evt : events) {
		for (Class<?> expectedClass: expectedClasses) {
			if (expectedClass.equals(TestTaskEvent.class)) {
				try {
					checkIsEvent(evt, expectedClass);
					fail("This should fail");
				}
				catch (UnsupportedOperationException ex) {
					// expected
				}
			}
			else if (evt.getClass().equals(expectedClass)) {
				assertTrue(checkIsEvent(evt, expectedClass));
			} else {
				assertFalse(checkIsEvent(evt, expectedClass));
			}
		}
	}
}
 

private static void assertNextBufferOrEvent(
		ResultSubpartitionView readView,
		int expectedReadableBufferSize,
		boolean expectedIsBuffer,
		@Nullable Class<? extends AbstractEvent> expectedEventClass,
		boolean expectedIsMoreAvailable,
		int expectedBuffersInBacklog,
		boolean expectedNextBufferIsEvent,
		boolean expectedRecycledAfterRecycle) throws IOException, InterruptedException {
	checkArgument(expectedEventClass == null || !expectedIsBuffer);

	ResultSubpartition.BufferAndBacklog bufferAndBacklog = readView.getNextBuffer();
	assertNotNull(bufferAndBacklog);
	try {
		assertEquals("buffer size", expectedReadableBufferSize,
				bufferAndBacklog.buffer().readableBytes());
		assertEquals("buffer or event", expectedIsBuffer,
				bufferAndBacklog.buffer().isBuffer());
		if (expectedEventClass != null) {
			Assert.assertThat(EventSerializer
							.fromBuffer(bufferAndBacklog.buffer(), ClassLoader.getSystemClassLoader()),
					instanceOf(expectedEventClass));
		}
		assertEquals("more available", expectedIsMoreAvailable,
				bufferAndBacklog.isMoreAvailable());
		assertEquals("more available", expectedIsMoreAvailable, readView.isAvailable());
		assertEquals("backlog", expectedBuffersInBacklog, bufferAndBacklog.buffersInBacklog());
		assertEquals("next is event", expectedNextBufferIsEvent,
				bufferAndBacklog.nextBufferIsEvent());
		assertEquals("next is event", expectedNextBufferIsEvent,
				readView.nextBufferIsEvent());

		assertFalse("not recycled", bufferAndBacklog.buffer().isRecycled());
	} finally {
		bufferAndBacklog.buffer().recycleBuffer();
	}
	assertEquals("recycled", expectedRecycledAfterRecycle, bufferAndBacklog.buffer().isRecycled());
}
 

private void processBufferOrEvent(BufferOrEvent bufferOrEvent) throws IOException {
	if (bufferOrEvent.isBuffer()) {
		lastChannel = bufferOrEvent.getChannelIndex();
		currentRecordDeserializer = recordDeserializers[lastChannel];
		currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
	}
	else {
		// Event received
		final AbstractEvent event = bufferOrEvent.getEvent();
		// TODO: with checkpointedInputGate.isFinished() we might not need to support any events on this level.
		if (event.getClass() != EndOfPartitionEvent.class) {
			throw new IOException("Unexpected event: " + event);
		}
	}
}
 

public void sendEvent(AbstractEvent event, int channel) {
	synchronized (inputQueues[channel]) {
		inputQueues[channel].add(InputValue.event(event));
		inputQueues[channel].notifyAll();
	}
	inputGate.notifyChannelNonEmpty(inputChannels[channel]);
}
 

private static String buildBufferTypeErrorMessage(Buffer buffer) {
	try {
		AbstractEvent event = EventSerializer.fromBuffer(buffer, ChannelStateWriterImpl.class.getClassLoader());
		return String.format("Should be buffer but [%s] found", event);
	}
	catch (Exception ex) {
		return "Should be buffer";
	}
}
 

public static ByteBuffer toSerializedEvent(AbstractEvent event) throws IOException {
	final Class<?> eventClass = event.getClass();
	if (eventClass == EndOfPartitionEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_PARTITION_EVENT });
	}
	else if (eventClass == CheckpointBarrier.class) {
		return serializeCheckpointBarrier((CheckpointBarrier) event);
	}
	else if (eventClass == EndOfSuperstepEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_SUPERSTEP_EVENT });
	}
	else if (eventClass == EndOfChannelStateEvent.class) {
		return ByteBuffer.wrap(new byte[] { 0, 0, 0, END_OF_CHANNEL_STATE_EVENT });
	}
	else if (eventClass == CancelCheckpointMarker.class) {
		CancelCheckpointMarker marker = (CancelCheckpointMarker) event;

		ByteBuffer buf = ByteBuffer.allocate(12);
		buf.putInt(0, CANCEL_CHECKPOINT_MARKER_EVENT);
		buf.putLong(4, marker.getCheckpointId());
		return buf;
	}
	else {
		try {
			final DataOutputSerializer serializer = new DataOutputSerializer(128);
			serializer.writeInt(OTHER_EVENT);
			serializer.writeUTF(event.getClass().getName());
			event.write(serializer);
			return serializer.wrapAsByteBuffer();
		}
		catch (IOException e) {
			throw new IOException("Error while serializing event.", e);
		}
	}
}
 

static BufferOrEvent parseBuffer(BufferConsumer bufferConsumer, int targetChannel) throws IOException {
	Buffer buffer = buildSingleBuffer(bufferConsumer);
	if (buffer.isBuffer()) {
		return new BufferOrEvent(buffer, new InputChannelInfo(0, targetChannel));
	} else {
		// is event:
		AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
		buffer.recycleBuffer(); // the buffer is not needed anymore
		return new BufferOrEvent(event, new InputChannelInfo(0, targetChannel));
	}
}
 

public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
	try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event)) {
		for (int targetChannel = 0; targetChannel < numberOfChannels; targetChannel++) {
			tryFinishCurrentBufferBuilder(targetChannel);

			// Retain the buffer so that it can be recycled by each channel of targetPartition
			targetPartition.addBufferConsumer(eventBufferConsumer.copy(), targetChannel, isPriorityEvent);
		}

		if (flushAlways) {
			flushAll();
		}
	}
}
 

private boolean isEndOfChannelStateEvent(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return false;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	buffer.setReaderIndex(0);
	return event.getClass() == EndOfChannelStateEvent.class;
}
 

/**
 * Parses the buffer as an event and returns the {@link CheckpointBarrier} if the event is indeed a barrier or
 * returns null in all other cases.
 */
@Nullable
protected CheckpointBarrier parseCheckpointBarrierOrNull(Buffer buffer) throws IOException {
	if (buffer.isBuffer()) {
		return null;
	}

	AbstractEvent event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	// reset the buffer because it would be deserialized again in SingleInputGate while getting next buffer.
	// we can further improve to avoid double deserialization in the future.
	buffer.setReaderIndex(0);
	return event.getClass() == CheckpointBarrier.class ? (CheckpointBarrier) event : null;
}
 

private BufferOrEvent transformEvent(
		Buffer buffer,
		boolean moreAvailable,
		InputChannel currentChannel) throws IOException, InterruptedException {
	final AbstractEvent event;
	try {
		event = EventSerializer.fromBuffer(buffer, getClass().getClassLoader());
	} finally {
		buffer.recycleBuffer();
	}

	if (event.getClass() == EndOfPartitionEvent.class) {
		channelsWithEndOfPartitionEvents.set(currentChannel.getChannelIndex());

		if (channelsWithEndOfPartitionEvents.cardinality() == numberOfInputChannels) {
			// Because of race condition between:
			// 1. releasing inputChannelsWithData lock in this method and reaching this place
			// 2. empty data notification that re-enqueues a channel
			// we can end up with moreAvailable flag set to true, while we expect no more data.
			checkState(!moreAvailable || !pollNext().isPresent());
			moreAvailable = false;
			hasReceivedAllEndOfPartitionEvents = true;
			markAvailable();
		}

		currentChannel.releaseAllResources();
	}

	return new BufferOrEvent(event, currentChannel.getChannelInfo(), moreAvailable, buffer.getSize());
}
 

public BufferOrEvent(AbstractEvent event, InputChannelInfo channelInfo, boolean moreAvailable, int size) {
	this.buffer = null;
	this.event = checkNotNull(event);
	this.channelInfo = channelInfo;
	this.moreAvailable = moreAvailable;
	this.size = size;
}
 

@Test
public void testSerializeDeserializeEvent() throws Exception {
	for (AbstractEvent evt : events) {
		ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
		assertTrue(serializedEvent.hasRemaining());

		AbstractEvent deserialized =
				EventSerializer.fromSerializedEvent(serializedEvent, getClass().getClassLoader());
		assertNotNull(deserialized);
		assertEquals(evt, deserialized);
	}
}