org.apache.flink.runtime.io.network.partition.consumer.LocalInputChannel Java Examples

@Test
public void testConsumptionWithLocalChannels() throws Exception {
	final int numberOfChannels = 11;
	final int buffersPerChannel = 1000;

	final ResultPartition resultPartition = mock(ResultPartition.class);

	final PipelinedSubpartition[] partitions = new PipelinedSubpartition[numberOfChannels];
	final Source[] sources = new Source[numberOfChannels];

	final ResultPartitionManager resultPartitionManager = createResultPartitionManager(partitions);

	final SingleInputGate gate = new SingleInputGate(
			"Test Task Name",
			new JobID(),
			new IntermediateDataSetID(), ResultPartitionType.PIPELINED,
			0, numberOfChannels,
			mock(TaskActions.class),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup(),
			true);

	for (int i = 0; i < numberOfChannels; i++) {
		LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(),
				resultPartitionManager, mock(TaskEventDispatcher.class), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());
		gate.setInputChannel(new IntermediateResultPartitionID(), channel);

		partitions[i] = new PipelinedSubpartition(0, resultPartition);
		sources[i] = new PipelinedSubpartitionSource(partitions[i]);
	}

	ProducerThread producer = new ProducerThread(sources, numberOfChannels * buffersPerChannel, 4, 10);
	ConsumerThread consumer = new ConsumerThread(gate, numberOfChannels * buffersPerChannel);
	producer.start();
	consumer.start();

	// the 'sync()' call checks for exceptions and failed assertions
	producer.sync();
	consumer.sync();
}
 

public static LocalInputChannel createLocalInputChannel(
	SingleInputGate inputGate,
	int channelIndex,
	ResultPartitionManager partitionManager) {

	return InputChannelBuilder.newBuilder()
		.setChannelIndex(channelIndex)
		.setPartitionManager(partitionManager)
		.buildLocalAndSetToGate(inputGate);
}
 

public static LocalInputChannel createLocalInputChannel(
	SingleInputGate inputGate,
	ResultPartitionManager partitionManager,
	int initialBackoff,
	int maxBackoff) {

	return InputChannelBuilder.newBuilder()
		.setPartitionManager(partitionManager)
		.setInitialBackoff(initialBackoff)
		.setMaxBackoff(maxBackoff)
		.buildLocalAndSetToGate(inputGate);
}
 

public static LocalInputChannel createLocalInputChannel(
	SingleInputGate inputGate,
	int channelIndex,
	ResultPartitionManager partitionManager) {

	return InputChannelBuilder.newBuilder()
		.setChannelIndex(channelIndex)
		.setPartitionManager(partitionManager)
		.buildLocalChannel(inputGate);
}
 

public static LocalInputChannel createLocalInputChannel(
	SingleInputGate inputGate,
	ResultPartitionManager partitionManager,
	int initialBackoff,
	int maxBackoff) {

	return InputChannelBuilder.newBuilder()
		.setPartitionManager(partitionManager)
		.setInitialBackoff(initialBackoff)
		.setMaxBackoff(maxBackoff)
		.buildLocalChannel(inputGate);
}
 

@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
	final int numberOfChannels = 37;
	final int buffersPerChannel = 27;

	final ResultPartition resultPartition = mock(ResultPartition.class);
	final BufferConsumer bufferConsumer = createFilledBufferConsumer(42);

	// ----- create some source channels and fill them with buffers -----

	final PipelinedSubpartition[] sources = new PipelinedSubpartition[numberOfChannels];

	for (int i = 0; i < numberOfChannels; i++) {
		PipelinedSubpartition partition = new PipelinedSubpartition(0, resultPartition);

		for (int p = 0; p < buffersPerChannel; p++) {
			partition.add(bufferConsumer.copy());
		}

		partition.finish();
		sources[i] = partition;
	}

	// ----- create reading side -----

	ResultPartitionManager resultPartitionManager = createResultPartitionManager(sources);

	SingleInputGate gate = new FairnessVerifyingInputGate(
			"Test Task Name",
			new JobID(),
			new IntermediateDataSetID(),
			0, numberOfChannels,
			mock(TaskActions.class),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup(),
			true);

	for (int i = 0; i < numberOfChannels; i++) {
		LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(),
				resultPartitionManager, mock(TaskEventDispatcher.class), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());
		gate.setInputChannel(new IntermediateResultPartitionID(), channel);
	}

	// read all the buffers and the EOF event
	for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
		assertNotNull(gate.getNextBufferOrEvent());

		int min = Integer.MAX_VALUE;
		int max = 0;

		for (PipelinedSubpartition source : sources) {
			int size = source.getCurrentNumberOfBuffers();
			min = Math.min(min, size);
			max = Math.max(max, size);
		}

		assertTrue(max == min || max == (min + 1));
	}

	assertFalse(gate.getNextBufferOrEvent().isPresent());
}
 

@Test
public void testFairConsumptionLocalChannels() throws Exception {
	final int numberOfChannels = 37;
	final int buffersPerChannel = 27;

	final ResultPartition resultPartition = mock(ResultPartition.class);
	try (BufferConsumer bufferConsumer = createFilledBufferConsumer(42)) {

		// ----- create some source channels and fill them with one buffer each -----

		final PipelinedSubpartition[] sources = new PipelinedSubpartition[numberOfChannels];

		for (int i = 0; i < numberOfChannels; i++) {
			sources[i] = new PipelinedSubpartition(0, resultPartition);
		}

		// ----- create reading side -----

		ResultPartitionManager resultPartitionManager = createResultPartitionManager(sources);

		SingleInputGate gate = new FairnessVerifyingInputGate(
			"Test Task Name",
			new JobID(),
			new IntermediateDataSetID(),
			0, numberOfChannels,
			mock(TaskActions.class),
			UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup(),
			true);

		for (int i = 0; i < numberOfChannels; i++) {
			LocalInputChannel channel = new LocalInputChannel(gate, i, new ResultPartitionID(),
				resultPartitionManager, mock(TaskEventDispatcher.class), UnregisteredMetricGroups.createUnregisteredTaskMetricGroup().getIOMetricGroup());
			gate.setInputChannel(new IntermediateResultPartitionID(), channel);
		}

		// seed one initial buffer
		sources[12].add(bufferConsumer.copy());

		// read all the buffers and the EOF event
		for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
			assertNotNull(gate.getNextBufferOrEvent());

			int min = Integer.MAX_VALUE;
			int max = 0;

			for (PipelinedSubpartition source : sources) {
				int size = source.getCurrentNumberOfBuffers();
				min = Math.min(min, size);
				max = Math.max(max, size);
			}

			assertTrue(max == min || max == min + 1);

			if (i % (2 * numberOfChannels) == 0) {
				// add three buffers to each channel, in random order
				fillRandom(sources, 3, bufferConsumer);
			}
		}
		// there is still more in the queues
	}
}
 

public static LocalInputChannel createLocalInputChannel(
	SingleInputGate inputGate,
	ResultPartitionManager partitionManager) {

	return createLocalInputChannel(inputGate, partitionManager, 0, 0);
}
 

public static LocalInputChannel createLocalInputChannel(
	SingleInputGate inputGate,
	ResultPartitionManager partitionManager) {

	return createLocalInputChannel(inputGate, partitionManager, 0, 0);
}