org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils Java Examples

The following examples show how to use org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReleasingReaderDoesNotReleasePartition() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());

	assertFalse(partition.isReleased());
	assertFalse(reader.isReleased());

	reader.releaseAllResources();

	assertTrue(reader.isReleased());
	assertFalse(partition.isReleased());

	partition.release();
}
 
Example #2
Source File: RecordWriterDelegateTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void verifyAvailability(RecordWriterDelegate writerDelegate) throws Exception {
	// writer is available at the beginning
	assertTrue(writerDelegate.isAvailable());
	assertTrue(writerDelegate.getAvailableFuture().isDone());

	// request one buffer from the local pool to make it unavailable
	RecordWriter recordWriter = writerDelegate.getRecordWriter(0);
	final BufferBuilder bufferBuilder = checkNotNull(recordWriter.getBufferBuilder(0));
	assertFalse(writerDelegate.isAvailable());
	CompletableFuture future = writerDelegate.getAvailableFuture();
	assertFalse(future.isDone());

	// recycle the buffer to make the local pool available again
	BufferBuilderTestUtils.fillBufferBuilder(bufferBuilder, 1).finish();
	ResultSubpartitionView readView = recordWriter.getTargetPartition().getSubpartition(0).createReadView(new NoOpBufferAvailablityListener());
	Buffer buffer = readView.getNextBuffer().buffer();

	buffer.recycleBuffer();
	assertTrue(future.isDone());
	assertTrue(writerDelegate.isAvailable());
	assertTrue(writerDelegate.getAvailableFuture().isDone());
}
 
Example #3
Source File: StreamTaskNetworkInputTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testIsAvailableWithBufferedDataInDeserializer() throws Exception {
	BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE);

	serializeRecord(42L, bufferBuilder);
	serializeRecord(44L, bufferBuilder);

	Buffer buffer = bufferBuilder.createBufferConsumer().build();

	List<BufferOrEvent> buffers = Collections.singletonList(new BufferOrEvent(buffer, 0, false));

	StreamTaskNetworkInput input = new StreamTaskNetworkInput(
		new CheckpointedInputGate(
			new MockInputGate(1, buffers, false),
			new EmptyBufferStorage(),
			new CheckpointBarrierTracker(1)),
		LongSerializer.INSTANCE,
		ioManager,
		0);

	assertHasNextElement(input);
	assertHasNextElement(input);
}
 
Example #4
Source File: BoundedDataTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void readInts(BoundedData.Reader reader, int numBuffersExpected, int numInts) throws IOException {
	Buffer b;
	int nextValue = 0;
	int numBuffers = 0;

	while ((b = reader.nextBuffer()) != null) {
		final int numIntsInBuffer = b.getSize() / 4;
		if (compressionEnabled && b.isCompressed()) {
			Buffer decompressedBuffer = DECOMPRESSOR.decompressToIntermediateBuffer(b);
			BufferBuilderTestUtils.validateBufferWithAscendingInts(decompressedBuffer, numIntsInBuffer, nextValue);
		} else {
			BufferBuilderTestUtils.validateBufferWithAscendingInts(b, numIntsInBuffer, nextValue);
		}
		nextValue += numIntsInBuffer;
		numBuffers++;

		b.recycleBuffer();
	}

	assertEquals(numBuffersExpected, numBuffers);
	assertThat(nextValue, Matchers.greaterThanOrEqualTo(numInts));
}
 
Example #5
Source File: BoundedDataTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static int writeInts(BoundedData bd, int numInts) throws IOException {
	final int numIntsInBuffer = BUFFER_SIZE / 4;
	int numBuffers = 0;

	for (int nextValue = 0; nextValue < numInts; nextValue += numIntsInBuffer) {
		Buffer buffer = BufferBuilderTestUtils.buildBufferWithAscendingInts(BUFFER_SIZE, numIntsInBuffer, nextValue);
		if (compressionEnabled) {
			bd.writeBuffer(COMPRESSOR.compressToIntermediateBuffer(buffer));
		} else {
			bd.writeBuffer(buffer);
		}
		numBuffers++;
	}

	return numBuffers;
}
 
Example #6
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testRecycleBufferAndConsumerOnFailure() throws Exception {
	final ResultSubpartition subpartition = createFailingWritesSubpartition();
	try {
		final BufferConsumer consumer = BufferBuilderTestUtils.createFilledBufferConsumer(100);

		try {
			subpartition.add(consumer);
			subpartition.flush();
			fail("should fail with an exception");
		}
		catch (Exception ignored) {
			// expected
		}

		assertTrue(consumer.isRecycled());
	}
	finally {
		subpartition.release();
	}
}
 
Example #7
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
	public void testReadAfterDispose() throws Exception {
		final ResultSubpartition partition = createSubpartition();
		partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
		partition.finish();

		final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());
		reader.releaseAllResources();

		// the reader must not throw an exception
		reader.getNextBuffer();

		// ideally, we want this to be null, but the pipelined partition still serves data
		// after dispose (which is unintuitive, but does not affect correctness)
//		assertNull(reader.getNextBuffer());
	}
 
Example #8
Source File: ResultPartitionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer(BufferConsumer, int)} on a working partition.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnPartition(final ResultPartitionType partitionType) throws Exception {
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	try {
		// partition.add() adds the bufferConsumer without recycling it (if not spilling)
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertFalse("bufferConsumer should not be recycled (still in the queue)", bufferConsumer.isRecycled());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
		}
		// should have been notified for pipelined partitions
		if (partitionType.isPipelined()) {
			verify(notifier, times(1))
				.notifyPartitionConsumable(eq(jobId), eq(consumableNotifyingPartitionWriter.getPartitionId()), eq(taskActions));
		}
	}
}
 
Example #9
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReleasingReaderDoesNotReleasePartition() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());

	assertFalse(partition.isReleased());
	assertFalse(reader.isReleased());

	reader.releaseAllResources();

	assertTrue(reader.isReleased());
	assertFalse(partition.isReleased());

	partition.release();
}
 
Example #10
Source File: ResultPartitionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer(BufferConsumer, int)} on a working partition.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnPartition(final ResultPartitionType partitionType) throws Exception {
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	try {
		// partition.add() adds the bufferConsumer without recycling it (if not spilling)
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertFalse("bufferConsumer should not be recycled (still in the queue)", bufferConsumer.isRecycled());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
		}
		// should have been notified for pipelined partitions
		if (partitionType.isPipelined()) {
			verify(notifier, times(1))
				.notifyPartitionConsumable(eq(jobId), eq(consumableNotifyingPartitionWriter.getPartitionId()), eq(taskActions));
		}
	}
}
 
Example #11
Source File: PipelinedSubpartitionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
	// Add a bufferConsumer
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	partition.add(bufferConsumer);
	partition.finish();

	// Create the view
	BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
	ResultSubpartitionView view = partition.createReadView(listener);

	// The added bufferConsumer and end-of-partition event
	assertNotNull(view.getNextBuffer());
	assertNotNull(view.getNextBuffer());

	// Release the parent
	assertFalse(view.isReleased());
	partition.release();

	// Verify that parent release is reflected at partition view
	assertTrue(view.isReleased());
}
 
Example #12
Source File: BoundedDataTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void readInts(BoundedData.Reader reader, int numBuffersExpected, int numInts) throws IOException {
	Buffer b;
	int nextValue = 0;
	int numBuffers = 0;

	while ((b = reader.nextBuffer()) != null) {
		final int numIntsInBuffer = b.getSize() / 4;
		BufferBuilderTestUtils.validateBufferWithAscendingInts(b, numIntsInBuffer, nextValue);
		nextValue += numIntsInBuffer;
		numBuffers++;

		b.recycleBuffer();
	}

	assertEquals(numBuffersExpected, numBuffers);
	assertThat(nextValue, Matchers.greaterThanOrEqualTo(numInts));
}
 
Example #13
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
	public void testReadAfterDispose() throws Exception {
		final ResultSubpartition partition = createSubpartition();
		partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
		partition.finish();

		final ResultSubpartitionView reader = partition.createReadView(new NoOpBufferAvailablityListener());
		reader.releaseAllResources();

		// the reader must not throw an exception
		reader.getNextBuffer();

		// ideally, we want this to be null, but the pipelined partition still serves data
		// after dispose (which is unintuitive, but does not affect correctness)
//		assertNull(reader.getNextBuffer());
	}
 
Example #14
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testRecycleBufferAndConsumerOnFailure() throws Exception {
	final ResultSubpartition subpartition = createFailingWritesSubpartition();
	try {
		final BufferConsumer consumer = BufferBuilderTestUtils.createFilledFinishedBufferConsumer(100);

		try {
			subpartition.add(consumer);
			subpartition.flush();
			fail("should fail with an exception");
		}
		catch (Exception ignored) {
			// expected
		}

		assertTrue(consumer.isRecycled());
	}
	finally {
		subpartition.release();
	}
}
 
Example #15
Source File: PipelinedSubpartitionTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
	// Add a bufferConsumer
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	partition.add(bufferConsumer);
	partition.finish();

	// Create the view
	BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
	ResultSubpartitionView view = partition.createReadView(listener);

	// The added bufferConsumer and end-of-partition event
	assertNotNull(view.getNextBuffer());
	assertNotNull(view.getNextBuffer());

	// Release the parent
	assertFalse(view.isReleased());
	partition.release();

	// Verify that parent release is reflected at partition view
	assertTrue(view.isReleased());
}
 
Example #16
Source File: SubpartitionTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
	// Add a bufferConsumer
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	partition.add(bufferConsumer);
	partition.finish();

	// Create the view
	BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
	ResultSubpartitionView view = partition.createReadView(listener);

	// The added bufferConsumer and end-of-partition event
	assertNotNull(view.getNextBuffer());
	assertNotNull(view.getNextBuffer());

	// Release the parent
	assertFalse(view.isReleased());
	partition.release();

	// Verify that parent release is reflected at partition view
	assertTrue(view.isReleased());
}
 
Example #17
Source File: ResultPartitionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer(BufferConsumer, int)} on a working partition.
 *
 * @param pipelined the result partition type to set up
 */
protected void testAddOnPartition(final ResultPartitionType pipelined)
	throws Exception {
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	ResultPartition partition = createPartition(notifier, pipelined, true);
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	try {
		// partition.add() adds the bufferConsumer without recycling it (if not spilling)
		partition.addBufferConsumer(bufferConsumer, 0);
		assertFalse("bufferConsumer should not be recycled (still in the queue)", bufferConsumer.isRecycled());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
		}
		// should have been notified for pipelined partitions
		if (pipelined.isPipelined()) {
			verify(notifier, times(1))
				.notifyPartitionConsumable(
					eq(partition.getJobId()),
					eq(partition.getPartitionId()),
					any(TaskActions.class));
		}
	}
}
 
Example #18
Source File: ResultPartitionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already been released.
 *
 * @param pipelined the result partition type to set up
 */
protected void testAddOnReleasedPartition(final ResultPartitionType pipelined)
	throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	try {
		ResultPartition partition = createPartition(notifier, pipelined, true);
		partition.release();
		// partition.add() silently drops the bufferConsumer but recycles it
		partition.addBufferConsumer(bufferConsumer, 0);
		assertTrue(partition.isReleased());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(any(JobID.class), any(ResultPartitionID.class), any(TaskActions.class));
	}
}
 
Example #19
Source File: ResultPartitionTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already finished.
 *
 * @param pipelined the result partition type to set up
 */
protected void testAddOnFinishedPartition(final ResultPartitionType pipelined)
	throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	try {
		ResultPartition partition = createPartition(notifier, pipelined, true);
		partition.finish();
		reset(notifier);
		// partition.add() should fail
		partition.addBufferConsumer(bufferConsumer, 0);
		Assert.fail("exception expected");
	} catch (IllegalStateException e) {
		// expected => ignored
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(any(JobID.class), any(ResultPartitionID.class), any(TaskActions.class));
	}
}
 
Example #20
Source File: LocalInputChannelTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static ResultSubpartitionView createResultSubpartitionView(boolean addBuffer) throws IOException {
	int bufferSize = 4096;
	ResultPartition parent = PartitionTestUtils.createPartition(
		ResultPartitionType.PIPELINED,
		NoOpFileChannelManager.INSTANCE,
		true,
		bufferSize);
	ResultSubpartition subpartition = parent.getAllPartitions()[0];
	if (addBuffer) {
		subpartition.add(BufferBuilderTestUtils.createFilledFinishedBufferConsumer(bufferSize));
	}
	return subpartition.createReadView(() -> {});
}
 
Example #21
Source File: BoundedDataTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void testGetSize(BoundedData bd, int bufferSize1, int bufferSize2) throws Exception {
	final int expectedSize1 = bufferSize1 + BufferReaderWriterUtil.HEADER_LENGTH;
	final int expectedSizeFinal = bufferSize1 + bufferSize2 + 2 * BufferReaderWriterUtil.HEADER_LENGTH;

	bd.writeBuffer(BufferBuilderTestUtils.buildSomeBuffer(bufferSize1));
	assertEquals(expectedSize1, bd.getSize());

	bd.writeBuffer(BufferBuilderTestUtils.buildSomeBuffer(bufferSize2));
	assertEquals(expectedSizeFinal, bd.getSize());

	bd.finishWrite();
	assertEquals(expectedSizeFinal, bd.getSize());
}
 
Example #22
Source File: StreamTaskNetworkInputTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private BufferOrEvent createDataBuffer() throws IOException {
	BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE);
	BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
	serializeRecord(42L, bufferBuilder);
	serializeRecord(44L, bufferBuilder);

	return new BufferOrEvent(bufferConsumer.build(), new InputChannelInfo(0, 0), false);
}
 
Example #23
Source File: ResultPartitionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already finished.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnFinishedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	try {
		consumableNotifyingPartitionWriter.finish();
		reset(notifier);
		// partition.add() should fail
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		Assert.fail("exception expected");
	} catch (IllegalStateException e) {
		// expected => ignored
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(
			eq(jobId),
			eq(consumableNotifyingPartitionWriter.getPartitionId()),
			eq(taskActions));
	}
}
 
Example #24
Source File: ResultPartitionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already been released.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnReleasedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartition partition = partitionType == ResultPartitionType.BLOCKING ?
		createPartition(partitionType, fileChannelManager) : createPartition(partitionType);
	ResultPartitionWriter consumableNotifyingPartitionWriter = ConsumableNotifyingResultPartitionWriterDecorator.decorate(
		Collections.singleton(PartitionTestUtils.createPartitionDeploymentDescriptor(partitionType)),
		new ResultPartitionWriter[] {partition},
		taskActions,
		jobId,
		notifier)[0];
	try {
		partition.release();
		// partition.add() silently drops the bufferConsumer but recycles it
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertTrue(partition.isReleased());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(eq(jobId), eq(partition.getPartitionId()), eq(taskActions));
	}
}
 
Example #25
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testReleaseIsIdempotent() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledFinishedBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	partition.release();
	partition.release();
	partition.release();
}
 
Example #26
Source File: BackPressureStatsTrackerImplITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public void invoke() throws Exception {
	final BufferBuilder bufferBuilder = testBufferPool.requestBufferBuilderBlocking();
	// Got a buffer, yay!
	BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder).recycleBuffer();

	Thread.currentThread().join();
}
 
Example #27
Source File: RecordWriterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the RecordWriter is available iif the respective LocalBufferPool has at-least one available buffer.
 */
@Test
public void testIsAvailableOrNot() throws Exception {
	// setup
	final NetworkBufferPool globalPool = new NetworkBufferPool(10, 128, 2);
	final BufferPool localPool = globalPool.createBufferPool(1, 1, null, 1, Integer.MAX_VALUE);
	final ResultPartitionWriter resultPartition = new ResultPartitionBuilder()
		.setBufferPoolFactory(p -> localPool)
		.build();
	resultPartition.setup();
	final ResultPartitionWriter partitionWrapper = new ConsumableNotifyingResultPartitionWriterDecorator(
		new NoOpTaskActions(),
		new JobID(),
		resultPartition,
		new NoOpResultPartitionConsumableNotifier());
	final RecordWriter recordWriter = createRecordWriter(partitionWrapper);

	try {
		// record writer is available because of initial available global pool
		assertTrue(recordWriter.getAvailableFuture().isDone());

		// request one buffer from the local pool to make it unavailable afterwards
		final BufferBuilder bufferBuilder = resultPartition.getBufferBuilder(0);
		assertNotNull(bufferBuilder);
		assertFalse(recordWriter.getAvailableFuture().isDone());

		// recycle the buffer to make the local pool available again
		final Buffer buffer = BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder);
		buffer.recycleBuffer();
		assertTrue(recordWriter.getAvailableFuture().isDone());
		assertEquals(recordWriter.AVAILABLE, recordWriter.getAvailableFuture());

	} finally {
		localPool.lazyDestroy();
		globalPool.destroy();
	}
}
 
Example #28
Source File: SubpartitionTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testReleaseIsIdempotent() throws Exception {
	final ResultSubpartition partition = createSubpartition();
	partition.add(createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE));
	partition.finish();

	partition.release();
	partition.release();
	partition.release();
}
 
Example #29
Source File: ResultPartitionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already been released.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnReleasedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartition partition = partitionType == ResultPartitionType.BLOCKING ?
		createPartition(partitionType, fileChannelManager) : createPartition(partitionType);
	ResultPartitionWriter consumableNotifyingPartitionWriter = ConsumableNotifyingResultPartitionWriterDecorator.decorate(
		Collections.singleton(PartitionTestUtils.createPartitionDeploymentDescriptor(partitionType)),
		new ResultPartitionWriter[] {partition},
		taskActions,
		jobId,
		notifier)[0];
	try {
		partition.release();
		// partition.add() silently drops the bufferConsumer but recycles it
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		assertTrue(partition.isReleased());
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(eq(jobId), eq(partition.getPartitionId()), eq(taskActions));
	}
}
 
Example #30
Source File: ResultPartitionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link ResultPartition#addBufferConsumer} on a partition which has already finished.
 *
 * @param partitionType the result partition type to set up
 */
private void testAddOnFinishedPartition(final ResultPartitionType partitionType) throws Exception {
	BufferConsumer bufferConsumer = createFilledBufferConsumer(BufferBuilderTestUtils.BUFFER_SIZE);
	ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
	JobID jobId = new JobID();
	TaskActions taskActions = new NoOpTaskActions();
	ResultPartitionWriter consumableNotifyingPartitionWriter = createConsumableNotifyingResultPartitionWriter(
		partitionType,
		taskActions,
		jobId,
		notifier);
	try {
		consumableNotifyingPartitionWriter.finish();
		reset(notifier);
		// partition.add() should fail
		consumableNotifyingPartitionWriter.addBufferConsumer(bufferConsumer, 0);
		Assert.fail("exception expected");
	} catch (IllegalStateException e) {
		// expected => ignored
	} finally {
		if (!bufferConsumer.isRecycled()) {
			bufferConsumer.close();
			Assert.fail("bufferConsumer not recycled");
		}
		// should not have notified either
		verify(notifier, never()).notifyPartitionConsumable(
			eq(jobId),
			eq(consumableNotifyingPartitionWriter.getPartitionId()),
			eq(taskActions));
	}
}