org.apache.flink.streaming.connectors.kafka.testutils.TestPartitionDiscoverer Java Examples

The following examples show how to use org.apache.flink.streaming.connectors.kafka.testutils.TestPartitionDiscoverer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractPartitionDiscovererTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitionsEqualConsumersFixedPartitions() throws Exception {
	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
		new KafkaTopicPartition(TEST_TOPIC, 0),
		new KafkaTopicPartition(TEST_TOPIC, 1),
		new KafkaTopicPartition(TEST_TOPIC, 2),
		new KafkaTopicPartition(TEST_TOPIC, 3));

	int numSubtasks = mockGetAllPartitionsForTopicsReturn.size();

	// get the start index; the assertions below will fail if the assignment logic does not meet correct contracts
	int numConsumers = KafkaTopicPartitionAssigner.assign(mockGetAllPartitionsForTopicsReturn.get(0), numSubtasks);

	for (int subtaskIndex = 0; subtaskIndex < mockGetAllPartitionsForTopicsReturn.size(); subtaskIndex++) {
		TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				mockGetAllPartitionsForTopicsReturn.size(),
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList(TEST_TOPIC)),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
		partitionDiscoverer.open();

		List<KafkaTopicPartition> initialDiscovery = partitionDiscoverer.discoverPartitions();
		assertEquals(1, initialDiscovery.size());
		assertTrue(contains(mockGetAllPartitionsForTopicsReturn, initialDiscovery.get(0).getPartition()));
		assertEquals(
			getExpectedSubtaskIndex(initialDiscovery.get(0), numConsumers, numSubtasks),
			subtaskIndex);

		// subsequent discoveries should not find anything
		List<KafkaTopicPartition> secondDiscovery = partitionDiscoverer.discoverPartitions();
		List<KafkaTopicPartition> thirdDiscovery = partitionDiscoverer.discoverPartitions();
		assertEquals(0, secondDiscovery.size());
		assertEquals(0, thirdDiscovery.size());
	}
}
 
Example #2
Source File: AbstractPartitionDiscovererTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testNonContiguousPartitionIdDiscovery() throws Exception {
	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn1 = Arrays.asList(
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 4));

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn2 = Arrays.asList(
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 2),
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic", 4));

	TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
			topicsDescriptor,
			0,
			1,
			TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList("test-topic")),
			// first metadata fetch has missing partitions that appears only in the second fetch;
			// need to create new modifiable lists for each fetch, since internally Iterable.remove() is used.
			Arrays.asList(new ArrayList<>(mockGetAllPartitionsForTopicsReturn1), new ArrayList<>(mockGetAllPartitionsForTopicsReturn2)));
	partitionDiscoverer.open();

	List<KafkaTopicPartition> discoveredPartitions1 = partitionDiscoverer.discoverPartitions();
	assertEquals(2, discoveredPartitions1.size());
	assertTrue(discoveredPartitions1.contains(new KafkaTopicPartition("test-topic", 1)));
	assertTrue(discoveredPartitions1.contains(new KafkaTopicPartition("test-topic", 4)));

	List<KafkaTopicPartition> discoveredPartitions2 = partitionDiscoverer.discoverPartitions();
	assertEquals(3, discoveredPartitions2.size());
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 0)));
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 2)));
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 3)));
}
 
Example #3
Source File: FlinkKafkaConsumerBaseTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void checkFilterRestoredPartitionsWithDisovered(
		List<String> restoredKafkaTopics,
		List<String> initKafkaTopics,
		List<String> expectedSubscribedPartitions,
		Boolean disableFiltering) throws Exception {
	final AbstractPartitionDiscoverer discoverer = new TestPartitionDiscoverer(
		new KafkaTopicsDescriptor(initKafkaTopics, null),
		0,
		1,
		TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(initKafkaTopics),
		TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(
			initKafkaTopics.stream()
				.map(topic -> new KafkaTopicPartition(topic, 0))
				.collect(Collectors.toList())));
	final FlinkKafkaConsumerBase<String> consumer = new DummyFlinkKafkaConsumer<>(initKafkaTopics, discoverer);
	if (disableFiltering) {
		consumer.disableFilterRestoredPartitionsWithSubscribedTopics();
	}

	final TestingListState<Tuple2<KafkaTopicPartition, Long>> listState = new TestingListState<>();

	for (int i = 0; i < restoredKafkaTopics.size(); i++) {
		listState.add(new Tuple2<>(new KafkaTopicPartition(restoredKafkaTopics.get(i), 0), 12345L));
	}

	setupConsumer(consumer, true, listState, true, 0, 1);

	Map<KafkaTopicPartition, Long> subscribedPartitionsToStartOffsets = consumer.getSubscribedPartitionsToStartOffsets();

	assertEquals(new HashSet<>(expectedSubscribedPartitions),
		subscribedPartitionsToStartOffsets
			.keySet()
			.stream()
			.map(partition -> partition.getTopic())
			.collect(Collectors.toSet()));
}
 
Example #4
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitionsEqualConsumersFixedPartitions() throws Exception {
	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
		new KafkaTopicPartition(TEST_TOPIC, 0),
		new KafkaTopicPartition(TEST_TOPIC, 1),
		new KafkaTopicPartition(TEST_TOPIC, 2),
		new KafkaTopicPartition(TEST_TOPIC, 3));

	int numSubtasks = mockGetAllPartitionsForTopicsReturn.size();

	// get the start index; the assertions below will fail if the assignment logic does not meet correct contracts
	int numConsumers = KafkaTopicPartitionAssigner.assign(mockGetAllPartitionsForTopicsReturn.get(0), numSubtasks);

	for (int subtaskIndex = 0; subtaskIndex < mockGetAllPartitionsForTopicsReturn.size(); subtaskIndex++) {
		TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				mockGetAllPartitionsForTopicsReturn.size(),
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList(TEST_TOPIC)),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
		partitionDiscoverer.open();

		List<KafkaTopicPartition> initialDiscovery = partitionDiscoverer.discoverPartitions();
		assertEquals(1, initialDiscovery.size());
		assertTrue(contains(mockGetAllPartitionsForTopicsReturn, initialDiscovery.get(0).getPartition()));
		assertEquals(
			getExpectedSubtaskIndex(initialDiscovery.get(0), numConsumers, numSubtasks),
			subtaskIndex);

		// subsequent discoveries should not find anything
		List<KafkaTopicPartition> secondDiscovery = partitionDiscoverer.discoverPartitions();
		List<KafkaTopicPartition> thirdDiscovery = partitionDiscoverer.discoverPartitions();
		assertEquals(0, secondDiscovery.size());
		assertEquals(0, thirdDiscovery.size());
	}
}
 
Example #5
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNonContiguousPartitionIdDiscovery() throws Exception {
	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn1 = Arrays.asList(
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 4));

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn2 = Arrays.asList(
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 2),
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic", 4));

	TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
			topicsDescriptor,
			0,
			1,
			TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList("test-topic")),
			// first metadata fetch has missing partitions that appears only in the second fetch;
			// need to create new modifiable lists for each fetch, since internally Iterable.remove() is used.
			Arrays.asList(new ArrayList<>(mockGetAllPartitionsForTopicsReturn1), new ArrayList<>(mockGetAllPartitionsForTopicsReturn2)));
	partitionDiscoverer.open();

	List<KafkaTopicPartition> discoveredPartitions1 = partitionDiscoverer.discoverPartitions();
	assertEquals(2, discoveredPartitions1.size());
	assertTrue(discoveredPartitions1.contains(new KafkaTopicPartition("test-topic", 1)));
	assertTrue(discoveredPartitions1.contains(new KafkaTopicPartition("test-topic", 4)));

	List<KafkaTopicPartition> discoveredPartitions2 = partitionDiscoverer.discoverPartitions();
	assertEquals(3, discoveredPartitions2.size());
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 0)));
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 2)));
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 3)));
}
 
Example #6
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void checkFilterRestoredPartitionsWithDisovered(
		List<String> restoredKafkaTopics,
		List<String> initKafkaTopics,
		List<String> expectedSubscribedPartitions,
		Boolean disableFiltering) throws Exception {
	final AbstractPartitionDiscoverer discoverer = new TestPartitionDiscoverer(
		new KafkaTopicsDescriptor(initKafkaTopics, null),
		0,
		1,
		TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(initKafkaTopics),
		TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(
			initKafkaTopics.stream()
				.map(topic -> new KafkaTopicPartition(topic, 0))
				.collect(Collectors.toList())));
	final FlinkKafkaConsumerBase<String> consumer = new DummyFlinkKafkaConsumer<>(initKafkaTopics, discoverer);
	if (disableFiltering) {
		consumer.disableFilterRestoredPartitionsWithSubscribedTopics();
	}

	final TestingListState<Tuple2<KafkaTopicPartition, Long>> listState = new TestingListState<>();

	for (int i = 0; i < restoredKafkaTopics.size(); i++) {
		listState.add(new Tuple2<>(new KafkaTopicPartition(restoredKafkaTopics.get(i), 0), 12345L));
	}

	setupConsumer(consumer, true, listState, true, 0, 1);

	Map<KafkaTopicPartition, Long> subscribedPartitionsToStartOffsets = consumer.getSubscribedPartitionsToStartOffsets();

	assertEquals(new HashSet<>(expectedSubscribedPartitions),
		subscribedPartitionsToStartOffsets
			.keySet()
			.stream()
			.map(partition -> partition.getTopic())
			.collect(Collectors.toSet()));
}
 
Example #7
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartitionsEqualConsumersFixedPartitions() throws Exception {
	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
		new KafkaTopicPartition(TEST_TOPIC, 0),
		new KafkaTopicPartition(TEST_TOPIC, 1),
		new KafkaTopicPartition(TEST_TOPIC, 2),
		new KafkaTopicPartition(TEST_TOPIC, 3));

	int numSubtasks = mockGetAllPartitionsForTopicsReturn.size();

	// get the start index; the assertions below will fail if the assignment logic does not meet correct contracts
	int numConsumers = KafkaTopicPartitionAssigner.assign(mockGetAllPartitionsForTopicsReturn.get(0), numSubtasks);

	for (int subtaskIndex = 0; subtaskIndex < mockGetAllPartitionsForTopicsReturn.size(); subtaskIndex++) {
		TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				mockGetAllPartitionsForTopicsReturn.size(),
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList(TEST_TOPIC)),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
		partitionDiscoverer.open();

		List<KafkaTopicPartition> initialDiscovery = partitionDiscoverer.discoverPartitions();
		assertEquals(1, initialDiscovery.size());
		assertTrue(contains(mockGetAllPartitionsForTopicsReturn, initialDiscovery.get(0).getPartition()));
		assertEquals(
			getExpectedSubtaskIndex(initialDiscovery.get(0), numConsumers, numSubtasks),
			subtaskIndex);

		// subsequent discoveries should not find anything
		List<KafkaTopicPartition> secondDiscovery = partitionDiscoverer.discoverPartitions();
		List<KafkaTopicPartition> thirdDiscovery = partitionDiscoverer.discoverPartitions();
		assertEquals(0, secondDiscovery.size());
		assertEquals(0, thirdDiscovery.size());
	}
}
 
Example #8
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testNonContiguousPartitionIdDiscovery() throws Exception {
	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn1 = Arrays.asList(
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 4));

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn2 = Arrays.asList(
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 2),
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic", 4));

	TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
			topicsDescriptor,
			0,
			1,
			TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList("test-topic")),
			// first metadata fetch has missing partitions that appears only in the second fetch;
			// need to create new modifiable lists for each fetch, since internally Iterable.remove() is used.
			Arrays.asList(new ArrayList<>(mockGetAllPartitionsForTopicsReturn1), new ArrayList<>(mockGetAllPartitionsForTopicsReturn2)));
	partitionDiscoverer.open();

	List<KafkaTopicPartition> discoveredPartitions1 = partitionDiscoverer.discoverPartitions();
	assertEquals(2, discoveredPartitions1.size());
	assertTrue(discoveredPartitions1.contains(new KafkaTopicPartition("test-topic", 1)));
	assertTrue(discoveredPartitions1.contains(new KafkaTopicPartition("test-topic", 4)));

	List<KafkaTopicPartition> discoveredPartitions2 = partitionDiscoverer.discoverPartitions();
	assertEquals(3, discoveredPartitions2.size());
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 0)));
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 2)));
	assertTrue(discoveredPartitions2.contains(new KafkaTopicPartition("test-topic", 3)));
}
 
Example #9
Source File: FlinkKafkaConsumerBaseTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void checkFilterRestoredPartitionsWithDisovered(
		List<String> restoredKafkaTopics,
		List<String> initKafkaTopics,
		List<String> expectedSubscribedPartitions,
		Boolean disableFiltering) throws Exception {
	final AbstractPartitionDiscoverer discoverer = new TestPartitionDiscoverer(
		new KafkaTopicsDescriptor(initKafkaTopics, null),
		0,
		1,
		TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(initKafkaTopics),
		TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(
			initKafkaTopics.stream()
				.map(topic -> new KafkaTopicPartition(topic, 0))
				.collect(Collectors.toList())));
	final FlinkKafkaConsumerBase<String> consumer = new DummyFlinkKafkaConsumer<>(initKafkaTopics, discoverer);
	if (disableFiltering) {
		consumer.disableFilterRestoredPartitionsWithSubscribedTopics();
	}

	final TestingListState<Tuple2<KafkaTopicPartition, Long>> listState = new TestingListState<>();

	for (int i = 0; i < restoredKafkaTopics.size(); i++) {
		listState.add(new Tuple2<>(new KafkaTopicPartition(restoredKafkaTopics.get(i), 0), 12345L));
	}

	setupConsumer(consumer, true, listState, true, 0, 1);

	Map<KafkaTopicPartition, Long> subscribedPartitionsToStartOffsets = consumer.getSubscribedPartitionsToStartOffsets();

	assertEquals(new HashSet<>(expectedSubscribedPartitions),
		subscribedPartitionsToStartOffsets
			.keySet()
			.stream()
			.map(partition -> partition.getTopic())
			.collect(Collectors.toSet()));
}
 
Example #10
Source File: AbstractPartitionDiscovererTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testPartitionsFewerThanConsumersFixedPartitions() {
	try {
		List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
			new KafkaTopicPartition(TEST_TOPIC, 0),
			new KafkaTopicPartition(TEST_TOPIC, 1),
			new KafkaTopicPartition(TEST_TOPIC, 2),
			new KafkaTopicPartition(TEST_TOPIC, 3));

		final Set<KafkaTopicPartition> allPartitions = new HashSet<>();
		allPartitions.addAll(mockGetAllPartitionsForTopicsReturn);

		final int numConsumers = 2 * mockGetAllPartitionsForTopicsReturn.size() + 3;

		// get the start index; the assertions below will fail if the assignment logic does not meet correct contracts
		int startIndex = KafkaTopicPartitionAssigner.assign(mockGetAllPartitionsForTopicsReturn.get(0), numConsumers);

		for (int subtaskIndex = 0; subtaskIndex < numConsumers; subtaskIndex++) {
			TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
					topicsDescriptor,
					subtaskIndex,
					numConsumers,
					TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList(TEST_TOPIC)),
					TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
			partitionDiscoverer.open();

			List<KafkaTopicPartition> initialDiscovery = partitionDiscoverer.discoverPartitions();
			assertTrue(initialDiscovery.size() <= 1);

			for (KafkaTopicPartition p : initialDiscovery) {
				// check that the element was actually contained
				assertTrue(allPartitions.remove(p));
				assertEquals(getExpectedSubtaskIndex(p, startIndex, numConsumers), subtaskIndex);
			}

			// subsequent discoveries should not find anything
			List<KafkaTopicPartition> secondDiscovery = partitionDiscoverer.discoverPartitions();
			List<KafkaTopicPartition> thirdDiscovery = partitionDiscoverer.discoverPartitions();
			assertEquals(0, secondDiscovery.size());
			assertEquals(0, thirdDiscovery.size());
		}

		// all partitions must have been assigned
		assertTrue(allPartitions.isEmpty());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #11
Source File: AbstractPartitionDiscovererTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeterministicAssignmentWithDifferentFetchedPartitionOrdering() throws Exception {
	int numSubtasks = 4;

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 2),
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic2", 0),
		new KafkaTopicPartition("test-topic2", 1));

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturnOutOfOrder = Arrays.asList(
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic2", 1),
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic2", 0),
		new KafkaTopicPartition("test-topic", 2));

	for (int subtaskIndex = 0; subtaskIndex < numSubtasks; subtaskIndex++) {
		TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				numSubtasks,
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Arrays.asList("test-topic", "test-topic2")),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
		partitionDiscoverer.open();

		TestPartitionDiscoverer partitionDiscovererOutOfOrder = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				numSubtasks,
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Arrays.asList("test-topic", "test-topic2")),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturnOutOfOrder));
		partitionDiscovererOutOfOrder.open();

		List<KafkaTopicPartition> discoveredPartitions = partitionDiscoverer.discoverPartitions();
		List<KafkaTopicPartition> discoveredPartitionsOutOfOrder = partitionDiscovererOutOfOrder.discoverPartitions();

		// the subscribed partitions should be identical, regardless of the input partition ordering
		Collections.sort(discoveredPartitions, new KafkaTopicPartition.Comparator());
		Collections.sort(discoveredPartitionsOutOfOrder, new KafkaTopicPartition.Comparator());
		assertEquals(discoveredPartitions, discoveredPartitionsOutOfOrder);
	}
}
 
Example #12
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testPartitionsFewerThanConsumersFixedPartitions() {
	try {
		List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
			new KafkaTopicPartition(TEST_TOPIC, 0),
			new KafkaTopicPartition(TEST_TOPIC, 1),
			new KafkaTopicPartition(TEST_TOPIC, 2),
			new KafkaTopicPartition(TEST_TOPIC, 3));

		final Set<KafkaTopicPartition> allPartitions = new HashSet<>();
		allPartitions.addAll(mockGetAllPartitionsForTopicsReturn);

		final int numConsumers = 2 * mockGetAllPartitionsForTopicsReturn.size() + 3;

		// get the start index; the assertions below will fail if the assignment logic does not meet correct contracts
		int startIndex = KafkaTopicPartitionAssigner.assign(mockGetAllPartitionsForTopicsReturn.get(0), numConsumers);

		for (int subtaskIndex = 0; subtaskIndex < numConsumers; subtaskIndex++) {
			TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
					topicsDescriptor,
					subtaskIndex,
					numConsumers,
					TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList(TEST_TOPIC)),
					TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
			partitionDiscoverer.open();

			List<KafkaTopicPartition> initialDiscovery = partitionDiscoverer.discoverPartitions();
			assertTrue(initialDiscovery.size() <= 1);

			for (KafkaTopicPartition p : initialDiscovery) {
				// check that the element was actually contained
				assertTrue(allPartitions.remove(p));
				assertEquals(getExpectedSubtaskIndex(p, startIndex, numConsumers), subtaskIndex);
			}

			// subsequent discoveries should not find anything
			List<KafkaTopicPartition> secondDiscovery = partitionDiscoverer.discoverPartitions();
			List<KafkaTopicPartition> thirdDiscovery = partitionDiscoverer.discoverPartitions();
			assertEquals(0, secondDiscovery.size());
			assertEquals(0, thirdDiscovery.size());
		}

		// all partitions must have been assigned
		assertTrue(allPartitions.isEmpty());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #13
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeterministicAssignmentWithDifferentFetchedPartitionOrdering() throws Exception {
	int numSubtasks = 4;

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 2),
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic2", 0),
		new KafkaTopicPartition("test-topic2", 1));

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturnOutOfOrder = Arrays.asList(
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic2", 1),
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic2", 0),
		new KafkaTopicPartition("test-topic", 2));

	for (int subtaskIndex = 0; subtaskIndex < numSubtasks; subtaskIndex++) {
		TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				numSubtasks,
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Arrays.asList("test-topic", "test-topic2")),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
		partitionDiscoverer.open();

		TestPartitionDiscoverer partitionDiscovererOutOfOrder = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				numSubtasks,
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Arrays.asList("test-topic", "test-topic2")),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturnOutOfOrder));
		partitionDiscovererOutOfOrder.open();

		List<KafkaTopicPartition> discoveredPartitions = partitionDiscoverer.discoverPartitions();
		List<KafkaTopicPartition> discoveredPartitionsOutOfOrder = partitionDiscovererOutOfOrder.discoverPartitions();

		// the subscribed partitions should be identical, regardless of the input partition ordering
		Collections.sort(discoveredPartitions, new KafkaTopicPartition.Comparator());
		Collections.sort(discoveredPartitionsOutOfOrder, new KafkaTopicPartition.Comparator());
		assertEquals(discoveredPartitions, discoveredPartitionsOutOfOrder);
	}
}
 
Example #14
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testPartitionsFewerThanConsumersFixedPartitions() {
	try {
		List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
			new KafkaTopicPartition(TEST_TOPIC, 0),
			new KafkaTopicPartition(TEST_TOPIC, 1),
			new KafkaTopicPartition(TEST_TOPIC, 2),
			new KafkaTopicPartition(TEST_TOPIC, 3));

		final Set<KafkaTopicPartition> allPartitions = new HashSet<>();
		allPartitions.addAll(mockGetAllPartitionsForTopicsReturn);

		final int numConsumers = 2 * mockGetAllPartitionsForTopicsReturn.size() + 3;

		// get the start index; the assertions below will fail if the assignment logic does not meet correct contracts
		int startIndex = KafkaTopicPartitionAssigner.assign(mockGetAllPartitionsForTopicsReturn.get(0), numConsumers);

		for (int subtaskIndex = 0; subtaskIndex < numConsumers; subtaskIndex++) {
			TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
					topicsDescriptor,
					subtaskIndex,
					numConsumers,
					TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Collections.singletonList(TEST_TOPIC)),
					TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
			partitionDiscoverer.open();

			List<KafkaTopicPartition> initialDiscovery = partitionDiscoverer.discoverPartitions();
			assertTrue(initialDiscovery.size() <= 1);

			for (KafkaTopicPartition p : initialDiscovery) {
				// check that the element was actually contained
				assertTrue(allPartitions.remove(p));
				assertEquals(getExpectedSubtaskIndex(p, startIndex, numConsumers), subtaskIndex);
			}

			// subsequent discoveries should not find anything
			List<KafkaTopicPartition> secondDiscovery = partitionDiscoverer.discoverPartitions();
			List<KafkaTopicPartition> thirdDiscovery = partitionDiscoverer.discoverPartitions();
			assertEquals(0, secondDiscovery.size());
			assertEquals(0, thirdDiscovery.size());
		}

		// all partitions must have been assigned
		assertTrue(allPartitions.isEmpty());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #15
Source File: AbstractPartitionDiscovererTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeterministicAssignmentWithDifferentFetchedPartitionOrdering() throws Exception {
	int numSubtasks = 4;

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturn = Arrays.asList(
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic", 2),
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic2", 0),
		new KafkaTopicPartition("test-topic2", 1));

	List<KafkaTopicPartition> mockGetAllPartitionsForTopicsReturnOutOfOrder = Arrays.asList(
		new KafkaTopicPartition("test-topic", 3),
		new KafkaTopicPartition("test-topic", 1),
		new KafkaTopicPartition("test-topic2", 1),
		new KafkaTopicPartition("test-topic", 0),
		new KafkaTopicPartition("test-topic2", 0),
		new KafkaTopicPartition("test-topic", 2));

	for (int subtaskIndex = 0; subtaskIndex < numSubtasks; subtaskIndex++) {
		TestPartitionDiscoverer partitionDiscoverer = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				numSubtasks,
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Arrays.asList("test-topic", "test-topic2")),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturn));
		partitionDiscoverer.open();

		TestPartitionDiscoverer partitionDiscovererOutOfOrder = new TestPartitionDiscoverer(
				topicsDescriptor,
				subtaskIndex,
				numSubtasks,
				TestPartitionDiscoverer.createMockGetAllTopicsSequenceFromFixedReturn(Arrays.asList("test-topic", "test-topic2")),
				TestPartitionDiscoverer.createMockGetAllPartitionsFromTopicsSequenceFromFixedReturn(mockGetAllPartitionsForTopicsReturnOutOfOrder));
		partitionDiscovererOutOfOrder.open();

		List<KafkaTopicPartition> discoveredPartitions = partitionDiscoverer.discoverPartitions();
		List<KafkaTopicPartition> discoveredPartitionsOutOfOrder = partitionDiscovererOutOfOrder.discoverPartitions();

		// the subscribed partitions should be identical, regardless of the input partition ordering
		Collections.sort(discoveredPartitions, new KafkaTopicPartition.Comparator());
		Collections.sort(discoveredPartitionsOutOfOrder, new KafkaTopicPartition.Comparator());
		assertEquals(discoveredPartitions, discoveredPartitionsOutOfOrder);
	}
}