Java Code Examples for org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils

The following examples show how to use org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: ResultPartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already finished.
 *
 * @param pipelined the result partition type to set up
 */
protected void testAddOnFinishedPartition(final ResultPartitionType pipelined)
	throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	try {
		ResultPartition partition = createPartition(notifier, pipelined, true);
		partition.finish();
		reset(notifier);
		// partition.add() should fail
		partition.addBufferConsumer(bufferConsumer, 0);
		Assert.fail("exception expected");
	} catch (IllegalStateException e) {
		// expected => ignored
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(any(JobID.class), any(ResultPartitionID.class), any(TaskActions.class));
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: ResultPartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already been released.
 *
 * @param pipelined the result partition type to set up
 */
protected void testAddOnReleasedPartition(final ResultPartitionType pipelined)
	throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	try {
		ResultPartition partition = createPartition(notifier, pipelined, true);
		partition.release();
		// partition.add() silently drops the bufferConsumer but recycles it
		partition.addBufferConsumer(bufferConsumer, 0);
		assertTrue(partition.isReleased());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(any(JobID.class), any(ResultPartitionID.class), any(TaskActions.class));
	}
}
 
Example 3
Source Project: Flink-CEPplus   Source File: ResultPartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer(BufferConsumer, int)} on a working partition.
 *
 * @param pipelined the result partition type to set up
 */
protected void testAddOnPartition(final ResultPartitionType pipelined)
	throws Exception {
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	ResultPartition partition = createPartition(notifier, pipelined, true);
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	try {
		// partition.add() adds the bufferConsumer without recycling it (if not spilling)
		partition.addBufferConsumer(bufferConsumer, 0);
		assertFalse("bufferConsumer should not be recycled (still in the queue)", bufferConsumer.isRecycled());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
		}
		// should have been notified for pipelined partitions
		if (pipelined.isPipelined()) {
			verify(notifier, times(1))
				.notifyPartitionConsumable(
					eq(partition.getJobId()),
					eq(partition.getPartitionId()),
					any(TaskActions.class));
		}
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
	// Add a bufferConsumer
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	partition.add(bufferConsumer);
	partition.finish();

	// Create the view
	BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
	ResultSubpartitionView view = partition.createReadView(listener);

	// The added bufferConsumer and end-of-partition event
	assertNotNull(view.getNextBuffer());
	assertNotNull(view.getNextBuffer());

	// Release the parent
	assertFalse(view.isReleased());
	partition.release();

	// Verify that parent release is reflected at partition view
	assertTrue(view.isReleased());
}
 
Example 5
Source Project: flink   Source File: PipelinedSubpartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
	// Add a bufferConsumer
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	partition.add(bufferConsumer);
	partition.finish();

	// Create the view
	BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
	ResultSubpartitionView view = partition.createReadView(listener);

	// The added bufferConsumer and end-of-partition event
	assertNotNull(view.getNextBuffer());
	assertNotNull(view.getNextBuffer());

	// Release the parent
	assertFalse(view.isReleased());
	partition.release();

	// Verify that parent release is reflected at partition view
	assertTrue(view.isReleased());
}
 
Example 6
Source Project: flink   Source File: BoundedDataTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private static void readInts(BoundedData.Reader reader, int numBuffersExpected, int numInts) throws IOException {
	Buffer b;
	int nextValue = 0;
	int numBuffers = 0;

	while ((b = reader.nextBuffer()) != null) {
		final int numIntsInBuffer = b.getSize() / 4;
		BufferBuilderTestUtils.validateBufferWithAscendingInts(b, numIntsInBuffer, nextValue);
		nextValue += numIntsInBuffer;
		numBuffers++;

		b.recycleBuffer();
	}

	assertEquals(numBuffersExpected, numBuffers);
	assertThat(nextValue, Matchers.greaterThanOrEqualTo(numInts));
}
 
Example 7
Source Project: flink   Source File: ResultPartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer(BufferConsumer, int)} on a working partition.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnPartition(final ResultPartitionType partitionType) throws Exception {
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	try {
		// partition.add() adds the bufferConsumer without recycling it (if not spilling)
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertFalse("bufferConsumer should not be recycled (still in the queue)", bufferConsumer.isRecycled());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
		}
		// should have been notified for pipelined partitions
		if (partitionType.isPipelined()) {
			verify(notifier, times(1))
				.notifyPartitionConsumable(eq(jobId), eq(consumableNotifyingPartitionWriter.getPartitionId()), eq(taskActions));
		}
	}
}
 
Example 8
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReleasingReaderDoesNotReleasePartition() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());

	assertFalse(partition.isReleased());
	assertFalse(reader.isReleased());

	reader.releaseAllResources();

	assertTrue(reader.isReleased());
	assertFalse(partition.isReleased());

	partition.release();
}
 
Example 9
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
	public void testReadAfterDispose() throws Exception {
		final ResultSubpartition partition = createSubpartition();
		partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
		partition.finish();

		final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());
		reader.releaseAllResources();

		// the reader must not throw an exception
		reader.getNextBuffer();

		// ideally, we want this to be null, but the pipelined partition still serves data
		// after dispose (which is unintuitive, but does not affect correctness)
//		assertNull(reader.getNextBuffer());
	}
 
Example 10
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecycleBufferAndConsumerOnFailure() throws Exception {
	final ResultSubpartition subpartition = createFailingWritesSubpartition();
	try {
		final BufferConsumer consumer = BufferBuilderTestUtils.createFilledBufferConsumer(100);

		try {
			subpartition.add(consumer);
			subpartition.flush();
			fail("should fail with an exception");
		}
		catch (Exception ignored) {
			// expected
		}

		assertTrue(consumer.isRecycled());
	}
	finally {
		subpartition.release();
	}
}
 
Example 11
Source Project: flink   Source File: StreamTaskNetworkInputTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIsAvailableWithBufferedDataInDeserializer() throws Exception {
	BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE);

	serializeRecord(42L, bufferBuilder);
	serializeRecord(44L, bufferBuilder);

	Buffer buffer = bufferBuilder.createBufferConsumer().build();

	List<BufferOrEvent> buffers = Collections.singletonList(new BufferOrEvent(buffer, 0, false));

	StreamTaskNetworkInput input = new StreamTaskNetworkInput(
		new CheckpointedInputGate(
			new MockInputGate(1, buffers, false),
			new EmptyBufferStorage(),
			new CheckpointBarrierTracker(1)),
		LongSerializer.INSTANCE,
		ioManager,
		0);

	assertHasNextElement(input);
	assertHasNextElement(input);
}
 
Example 12
Source Project: flink   Source File: RecordWriterDelegateTest.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyAvailability(RecordWriterDelegate writerDelegate) throws Exception {
	// writer is available at the beginning
	assertTrue(writerDelegate.isAvailable());
	assertTrue(writerDelegate.getAvailableFuture().isDone());

	// request one buffer from the local pool to make it unavailable
	RecordWriter recordWriter = writerDelegate.getRecordWriter(0);
	final BufferBuilder bufferBuilder = checkNotNull(recordWriter.getBufferBuilder(0));
	assertFalse(writerDelegate.isAvailable());
	CompletableFuture future = writerDelegate.getAvailableFuture();
	assertFalse(future.isDone());

	// recycle the buffer to make the local pool available again
	BufferBuilderTestUtils.fillBufferBuilder(bufferBuilder, 1).finish();
	ResultSubpartitionView readView = recordWriter.getTargetPartition().getSubpartition(0).createReadView(new NoOpBufferAvailablityListener());
	Buffer buffer = readView.getNextBuffer().buffer();

	buffer.recycleBuffer();
	assertTrue(future.isDone());
	assertTrue(writerDelegate.isAvailable());
	assertTrue(writerDelegate.getAvailableFuture().isDone());
}
 
Example 13
Source Project: flink   Source File: PipelinedSubpartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
	// Add a bufferConsumer
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	partition.add(bufferConsumer);
	partition.finish();

	// Create the view
	BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
	ResultSubpartitionView view = partition.createReadView(listener);

	// The added bufferConsumer and end-of-partition event
	assertNotNull(view.getNextBuffer());
	assertNotNull(view.getNextBuffer());

	// Release the parent
	assertFalse(view.isReleased());
	partition.release();

	// Verify that parent release is reflected at partition view
	assertTrue(view.isReleased());
}
 
Example 14
Source Project: flink   Source File: BoundedDataTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private static int writeInts(BoundedData bd, int numInts) throws IOException {
	final int numIntsInBuffer = BUFFER_SIZE / 4;
	int numBuffers = 0;

	for (int nextValue = 0; nextValue < numInts; nextValue += numIntsInBuffer) {
		Buffer buffer = BufferBuilderTestUtils.buildBufferWithAscendingInts(BUFFER_SIZE, numIntsInBuffer, nextValue);
		if (compressionEnabled) {
			bd.writeBuffer(COMPRESSOR.compressToIntermediateBuffer(buffer));
		} else {
			bd.writeBuffer(buffer);
		}
		numBuffers++;
	}

	return numBuffers;
}
 
Example 15
Source Project: flink   Source File: BoundedDataTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private static void readInts(BoundedData.Reader reader, int numBuffersExpected, int numInts) throws IOException {
	Buffer b;
	int nextValue = 0;
	int numBuffers = 0;

	while ((b = reader.nextBuffer()) != null) {
		final int numIntsInBuffer = b.getSize() / 4;
		if (compressionEnabled && b.isCompressed()) {
			Buffer decompressedBuffer = DECOMPRESSOR.decompressToIntermediateBuffer(b);
			BufferBuilderTestUtils.validateBufferWithAscendingInts(decompressedBuffer, numIntsInBuffer, nextValue);
		} else {
			BufferBuilderTestUtils.validateBufferWithAscendingInts(b, numIntsInBuffer, nextValue);
		}
		nextValue += numIntsInBuffer;
		numBuffers++;

		b.recycleBuffer();
	}

	assertEquals(numBuffersExpected, numBuffers);
	assertThat(nextValue, Matchers.greaterThanOrEqualTo(numInts));
}
 
Example 16
Source Project: flink   Source File: ResultPartitionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer(BufferConsumer, int)} on a working partition.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnPartition(final ResultPartitionType partitionType) throws Exception {
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	try {
		// partition.add() adds the bufferConsumer without recycling it (if not spilling)
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertFalse("bufferConsumer should not be recycled (still in the queue)", bufferConsumer.isRecycled());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
		}
		// should have been notified for pipelined partitions
		if (partitionType.isPipelined()) {
			verify(notifier, times(1))
				.notifyPartitionConsumable(eq(jobId), eq(consumableNotifyingPartitionWriter.getPartitionId()), eq(taskActions));
		}
	}
}
 
Example 17
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReleasingReaderDoesNotReleasePartition() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());

	assertFalse(partition.isReleased());
	assertFalse(reader.isReleased());

	reader.releaseAllResources();

	assertTrue(reader.isReleased());
	assertFalse(partition.isReleased());

	partition.release();
}
 
Example 18
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
	public void testReadAfterDispose() throws Exception {
		final ResultSubpartition partition = createSubpartition();
		partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
		partition.finish();

		final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());
		reader.releaseAllResources();

		// the reader must not throw an exception
		reader.getNextBuffer();

		// ideally, we want this to be null, but the pipelined partition still serves data
		// after dispose (which is unintuitive, but does not affect correctness)
//		assertNull(reader.getNextBuffer());
	}
 
Example 19
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecycleBufferAndConsumerOnFailure() throws Exception {
	final ResultSubpartition subpartition = createFailingWritesSubpartition();
	try {
		final BufferConsumer consumer = BufferBuilderTestUtils.createFilledFinishedBufferConsumer(100);

		try {
			subpartition.add(consumer);
			subpartition.flush();
			fail("should fail with an exception");
		}
		catch (Exception ignored) {
			// expected
		}

		assertTrue(consumer.isRecycled());
	}
	finally {
		subpartition.release();
	}
}
 
Example 20
@Override
public void invoke() throws Exception {
	final BufferBuilder bufferBuilder = testBufferPool.requestBufferBuilderBlocking();
	// Got a buffer, yay!
	BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder).recycleBuffer();

	Thread.currentThread().join();
}
 
Example 21
Source Project: Flink-CEPplus   Source File: InputGateConcurrentTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void go() throws Exception {
	final BufferConsumer bufferConsumer = BufferBuilderTestUtils.createFilledBufferConsumer(100);
	int nextYield = numTotal - yieldAfter;

	for (int i = numTotal; i > 0;) {
		final int nextChannel = rnd.nextInt(sources.length);
		final int chunk = Math.min(i, rnd.nextInt(maxChunk) + 1);

		final Source next = sources[nextChannel];

		for (int k = chunk; k > 0; --k) {
			next.addBufferConsumer(bufferConsumer.copy());
		}

		i -= chunk;

		if (i <= nextYield) {
			nextYield -= yieldAfter;
			//noinspection CallToThreadYield
			Thread.yield();
		}
	}

	for (Source source : sources) {
		source.flush();
	}
}
 
Example 22
Source Project: flink   Source File: BoundedDataTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private static int writeInts(BoundedData bd, int numInts) throws IOException {
	final int numIntsInBuffer = BUFFER_SIZE / 4;
	int numBuffers = 0;

	for (int nextValue = 0; nextValue < numInts; nextValue += numIntsInBuffer) {
		Buffer buffer = BufferBuilderTestUtils.buildBufferWithAscendingInts(BUFFER_SIZE, numIntsInBuffer, nextValue);
		bd.writeBuffer(buffer);
		numBuffers++;
	}

	return numBuffers;
}
 
Example 23
Source Project: flink   Source File: ResultPartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already finished.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnFinishedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	try {
		consumableNotifyingPartitionWriter.finish();
		reset(notifier);
		// partition.add() should fail
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		Assert.fail("exception expected");
	} catch (IllegalStateException e) {
		// expected => ignored
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(
			eq(jobId),
			eq(consumableNotifyingPartitionWriter.getPartitionId()),
			eq(taskActions));
	}
}
 
Example 24
Source Project: flink   Source File: ResultPartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already been released.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnReleasedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartition partition = partitionType == ResultPartitionType.BLOCKING ?
		createPartition(partitionType, fileChannelManager) : createPartition(partitionType);
	ResultPartitionWriter consumableNotifyingPartitionWriter = ConsumableNotifyingResultPartitionWriterDecorator.decorate(
		Collections.singleton(PartitionTestUtils.createPartitionDeploymentDescriptor(partitionType)),
		new ResultPartitionWriter[] {partition},
		taskActions,
		jobId,
		notifier)[0];
	try {
		partition.release();
		// partition.add() silently drops the bufferConsumer but recycles it
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertTrue(partition.isReleased());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(eq(jobId), eq(partition.getPartitionId()), eq(taskActions));
	}
}
 
Example 25
Source Project: flink   Source File: SubpartitionTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReleaseIsIdempotent() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	partition.release();
	partition.release();
	partition.release();
}
 
Example 26
Source Project: flink   Source File: RecordWriterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the RecordWriter is available iif the respective LocalBufferPool has at-least one available buffer.
 */
@Test
public void testIsAvailableOrNot() throws Exception {
	// setup
	final NetworkBufferPool globalPool = new NetworkBufferPool(10, 128, 2);
	final BufferPool localPool = globalPool.createBufferPool(1, 1, null, 1, Integer.MAX_VALUE);
	final ResultPartitionWriter resultPartition = new ResultPartitionBuilder()
		.setBufferPoolFactory(p -> localPool)
		.build();
	resultPartition.setup();
	final ResultPartitionWriter partitionWrapper = new ConsumableNotifyingResultPartitionWriterDecorator(
		new NoOpTaskActions(),
		new JobID(),
		resultPartition,
		new NoOpResultPartitionConsumableNotifier());
	final RecordWriter recordWriter = createRecordWriter(partitionWrapper);

	try {
		// record writer is available because of initial available global pool
		assertTrue(recordWriter.getAvailableFuture().isDone());

		// request one buffer from the local pool to make it unavailable afterwards
		final BufferBuilder bufferBuilder = resultPartition.getBufferBuilder(0);
		assertNotNull(bufferBuilder);
		assertFalse(recordWriter.getAvailableFuture().isDone());

		// recycle the buffer to make the local pool available again
		final Buffer buffer = BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder);
		buffer.recycleBuffer();
		assertTrue(recordWriter.getAvailableFuture().isDone());
		assertEquals(recordWriter.AVAILABLE, recordWriter.getAvailableFuture());

	} finally {
		localPool.lazyDestroy();
		globalPool.destroy();
	}
}
 
Example 27
Source Project: flink   Source File: LocalInputChannelTest.java    License: Apache License 2.0 5 votes vote down vote up
private static ResultSubpartitionView createResultSubpartitionView(boolean addBuffer) throws IOException {
	int bufferSize = 4096;
	ResultPartition parent = PartitionTestUtils.createPartition(
		ResultPartitionType.PIPELINED,
		NoOpFileChannelManager.INSTANCE,
		true,
		bufferSize);
	ResultSubpartition subpartition = parent.getAllPartitions()[0];
	if (addBuffer) {
		subpartition.add(BufferBuilderTestUtils.createFilledFinishedBufferConsumer(bufferSize));
	}
	return subpartition.createReadView(() -> {});
}
 
Example 28
Source Project: flink   Source File: BoundedDataTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private static void testGetSize(BoundedData bd, int bufferSize1, int bufferSize2) throws Exception {
	final int expectedSize1 = bufferSize1 + BufferReaderWriterUtil.HEADER_LENGTH;
	final int expectedSizeFinal = bufferSize1 + bufferSize2 + 2 * BufferReaderWriterUtil.HEADER_LENGTH;

	bd.writeBuffer(BufferBuilderTestUtils.buildSomeBuffer(bufferSize1));
	assertEquals(expectedSize1, bd.getSize());

	bd.writeBuffer(BufferBuilderTestUtils.buildSomeBuffer(bufferSize2));
	assertEquals(expectedSizeFinal, bd.getSize());

	bd.finishWrite();
	assertEquals(expectedSizeFinal, bd.getSize());
}
 
Example 29
Source Project: flink   Source File: ResultPartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already finished.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnFinishedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	try {
		consumableNotifyingPartitionWriter.finish();
		reset(notifier);
		// partition.add() should fail
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		Assert.fail("exception expected");
	} catch (IllegalStateException e) {
		// expected => ignored
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(
			eq(jobId),
			eq(consumableNotifyingPartitionWriter.getPartitionId()),
			eq(taskActions));
	}
}
 
Example 30
Source Project: flink   Source File: ResultPartitionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already been released.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnReleasedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartition partition = partitionType == ResultPartitionType.BLOCKING ?
		createPartition(partitionType, fileChannelManager) : createPartition(partitionType);
	ResultPartitionWriter consumableNotifyingPartitionWriter = ConsumableNotifyingResultPartitionWriterDecorator.decorate(
		Collections.singleton(PartitionTestUtils.createPartitionDeploymentDescriptor(partitionType)),
		new ResultPartitionWriter[] {partition},
		taskActions,
		jobId,
		notifier)[0];
	try {
		partition.release();
		// partition.add() silently drops the bufferConsumer but recycles it
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertTrue(partition.isReleased());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(eq(jobId), eq(partition.getPartitionId()), eq(taskActions));
	}
}