org.apache.flink.runtime.io.network.buffer.BufferListener Java Examples

/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 

/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 

/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 

/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final BufferResponse receivedBuffer = createBufferResponse(
			TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 

/**
 * Tests a fix for FLINK-1627.
 *
 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
 * provider of the receiving input channel, but the respective input channel is released (and
 * the corresponding buffer provider destroyed), the handler did not notice this.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
	// Mocks an input channel in a state as it was released during a decode.
	final BufferProvider bufferProvider = mock(BufferProvider.class);
	when(bufferProvider.requestBuffer()).thenReturn(null);
	when(bufferProvider.isDestroyed()).thenReturn(true);
	when(bufferProvider.addBufferListener(any(BufferListener.class))).thenReturn(false);

	final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
	when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
	when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);

	final CreditBasedPartitionRequestClientHandler client = new CreditBasedPartitionRequestClientHandler();
	client.addInputChannel(inputChannel);

	final BufferResponse receivedBuffer = createBufferResponse(
		TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE),
		0,
		inputChannel.getInputChannelId(),
		2,
		new NetworkBufferAllocator(client));

	client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
}
 

@Override
public void recycle(MemorySegment segment) {
	synchronized (listenerRegistrationLock) {
		final Buffer buffer = new NetworkBuffer(segment, this);

		BufferListener listener = registeredListeners.poll();

		if (listener == null) {
			buffers.add(buffer);
		}
		else {
			listener.notifyBufferAvailable(buffer);
		}
	}
}
 

boolean registerListener(BufferListener listener) {
	synchronized (listenerRegistrationLock) {
		if (buffers.isEmpty()) {
			registeredListeners.add(listener);

			return true;
		}

		return false;
	}
}
 

@Override
public void recycle(MemorySegment segment) {
	synchronized (listenerRegistrationLock) {
		final Buffer buffer = new NetworkBuffer(segment, this);

		BufferListener listener = registeredListeners.poll();

		if (listener == null) {
			buffers.add(buffer);
		}
		else {
			listener.notifyBufferAvailable(buffer);
		}
	}
}
 

boolean registerListener(BufferListener listener) {
	synchronized (listenerRegistrationLock) {
		if (buffers.isEmpty()) {
			registeredListeners.add(listener);

			return true;
		}

		return false;
	}
}
 

/**
 * The buffer pool notifies this listener of an available floating buffer. If the listener is released or
 * currently does not need extra buffers, the buffer should be returned to the buffer pool. Otherwise,
 * the buffer will be added into the <tt>bufferQueue</tt>.
 *
 * @param buffer Buffer that becomes available in buffer pool.
 * @return NotificationResult indicates whether this channel accepts the buffer and is waiting for
 * more floating buffers.
 */
@Override
public BufferListener.NotificationResult notifyBufferAvailable(Buffer buffer) {
	BufferListener.NotificationResult notificationResult = BufferListener.NotificationResult.BUFFER_NOT_USED;
	try {
		synchronized (bufferQueue) {
			checkState(isWaitingForFloatingBuffers, "This channel should be waiting for floating buffers.");

			// Important: make sure that we never add a buffer after releaseAllResources()
			// released all buffers. Following scenarios exist:
			// 1) releaseAllBuffers() already released buffers inside bufferQueue
			// -> while isReleased is set correctly in InputChannel
			// 2) releaseAllBuffers() did not yet release buffers from bufferQueue
			// -> we may or may not have set isReleased yet but will always wait for the
			// lock on bufferQueue to release buffers
			if (inputChannel.isReleased() || bufferQueue.getAvailableBufferSize() >= numRequiredBuffers) {
				isWaitingForFloatingBuffers = false;
				return notificationResult;
			}

			bufferQueue.addFloatingBuffer(buffer);
			bufferQueue.notifyAll();

			if (bufferQueue.getAvailableBufferSize() == numRequiredBuffers) {
				isWaitingForFloatingBuffers = false;
				notificationResult = BufferListener.NotificationResult.BUFFER_USED_NO_NEED_MORE;
			} else {
				notificationResult = BufferListener.NotificationResult.BUFFER_USED_NEED_MORE;
			}
		}

		if (notificationResult != NotificationResult.BUFFER_NOT_USED) {
			inputChannel.notifyBufferAvailable(1);
		}
	} catch (Throwable t) {
		inputChannel.setError(t);
	}

	return notificationResult;
}
 

@Override
public void recycle(MemorySegment segment) {
	synchronized (listenerRegistrationLock) {
		final Buffer buffer = new NetworkBuffer(segment, this);

		BufferListener listener = registeredListeners.poll();

		if (listener == null) {
			buffers.add(buffer);
		}
		else {
			listener.notifyBufferAvailable(buffer);
		}
	}
}
 

boolean registerListener(BufferListener listener) {
	synchronized (listenerRegistrationLock) {
		if (buffers.isEmpty()) {
			registeredListeners.add(listener);

			return true;
		}

		return false;
	}
}
 

@Override
public boolean addBufferListener(BufferListener listener) {
	return bufferRecycler.registerListener(listener);
}
 

@Override
public boolean addBufferListener(BufferListener listener) {
	return bufferRecycler.registerListener(listener);
}
 

@Override
public boolean addBufferListener(BufferListener listener) {
	return bufferRecycler.registerListener(listener);
}