org.apache.kafka.clients.admin.NewPartitions Java Examples

@Test
public void testIncreasePartitions() {
  TopicService service = new TopicServiceImpl(adminClient, true);
  CreatePartitionsResult result = mock(CreatePartitionsResult.class);
  when(result.all()).thenReturn(KafkaFuture.completedFuture(null));
  when(adminClient.createPartitions(any(Map.class), any(CreatePartitionsOptions.class))).thenReturn(result);

  service.increasePartitions(Collections.singletonList(
      new ConfiguredTopic("test", 3, (short) 2, Collections.emptyMap())));

  ArgumentCaptor<Map> increase = ArgumentCaptor.forClass(Map.class);
  ArgumentCaptor<CreatePartitionsOptions> options = ArgumentCaptor.forClass(CreatePartitionsOptions.class);
  verify(adminClient).createPartitions((Map<String, NewPartitions>) increase.capture(), options.capture());
  Assert.assertEquals(1, increase.getValue().size());
  Assert.assertTrue(increase.getValue().containsKey("test"));
  Assert.assertEquals(3, ((NewPartitions) increase.getValue().get("test")).totalCount());
  Assert.assertTrue(options.getValue().validateOnly());
}
 

protected void maybeIncreaseTopicPartitionCount() {
  String cruiseControlMetricsTopic = _metricsTopic.name();
  try {
    // Retrieve topic partition count to check and update.
    TopicDescription topicDescription =
        _adminClient.describeTopics(Collections.singletonList(cruiseControlMetricsTopic)).values()
                    .get(cruiseControlMetricsTopic).get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    if (topicDescription.partitions().size() < _metricsTopic.numPartitions()) {
      _adminClient.createPartitions(Collections.singletonMap(cruiseControlMetricsTopic,
                                                             NewPartitions.increaseTo(_metricsTopic.numPartitions())));
    }
  } catch (InterruptedException | ExecutionException | TimeoutException e) {
    LOG.warn("Partition count increase to {} for topic {} failed{}.", _metricsTopic.numPartitions(), cruiseControlMetricsTopic,
             (e.getCause() instanceof ReassignmentInProgressException) ? " due to ongoing reassignment" : "", e);
  }
}
 

protected void alterTopicNumPartitions(String topicName, String resourceName) throws InterruptedException, ExecutionException, TimeoutException {
    int changedValue = 2;

    NewPartitions newPartitions = NewPartitions.increaseTo(changedValue);
    Map<String, NewPartitions> map = new HashMap<>(1);
    map.put(topicName, newPartitions);

    CreatePartitionsResult createPartitionsResult = adminClient.createPartitions(map);
    createPartitionsResult.all().get();

    // Wait for the resource to be modified
    waitFor(() -> {
        KafkaTopic topic = operation().inNamespace(NAMESPACE).withName(resourceName).get();
        LOGGER.info("Polled topic {}, waiting for partitions change", resourceName);
        int gotValue = TopicSerialization.fromTopicResource(topic).getNumPartitions();
        LOGGER.info("Expected value {}, got value {}", changedValue, gotValue);
        return changedValue == gotValue;
    }, "Expected the topic " + topicName + "to have " + changedValue + " partitions by now");
}
 

@Override
public void repartition(final String topic, final int partitionsNumber) throws CannotAddPartitionToTopicException,
        TopicConfigException {
    try (AdminClient adminClient = AdminClient.create(kafkaLocationManager.getProperties())) {
        adminClient.createPartitions(ImmutableMap.of(topic, NewPartitions.increaseTo(partitionsNumber)));
        final long timeoutMillis = TimeUnit.SECONDS.toMillis(5);
        final Boolean areNewPartitionsAdded = Retryer.executeWithRetry(() -> {
                    try (Consumer<byte[], byte[]> consumer = kafkaFactory.getConsumer()) {
                        return consumer.partitionsFor(topic).size() == partitionsNumber;
                    }
                },
                new RetryForSpecifiedTimeStrategy<Boolean>(timeoutMillis)
                        .withWaitBetweenEachTry(100L)
                        .withResultsThatForceRetry(Boolean.FALSE));
        if (!Boolean.TRUE.equals(areNewPartitionsAdded)) {
            throw new TopicConfigException(String.format("Failed to repartition topic to %s", partitionsNumber));
        }
        final Producer<String, String> producer = kafkaFactory.takeProducer();
        kafkaFactory.terminateProducer(producer);
        kafkaFactory.releaseProducer(producer);
    } catch (Exception e) {
        throw new CannotAddPartitionToTopicException(String
                .format("Failed to increase the number of partition for %s topic to %s", topic,
                        partitionsNumber), e);
    }
}
 

public void createTopic(String topic, String numPartitions) throws Exception {
    // Create the testAT topic
    Map<String, String> topicProperties = new HashMap<String, String>() {
        {
            put(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1");
        }
    };

    logger.debug("Creating topic: " + topic);
    NewTopic newTopic = new NewTopic(topic, 1, (short) 1).configs(topicProperties);
    adminClient.createTopics(asList(newTopic)).all().get(KAFKA_DEFAULT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    logger.debug("Topic created.");

    if (numPartitions != null) {
        // create partitions
        logger.debug("Creating: " + numPartitions + " partitions in topic: " + topic);
        Map<String, NewPartitions> partitions = new HashMap<String, NewPartitions>() {
            {
                put (topic, NewPartitions.increaseTo(Integer.parseInt(numPartitions)));
            }
        };

        adminClient.createPartitions(partitions).all().get(KAFKA_DEFAULT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        logger.debug("Partitions created.");
    }
}
 

@Override
public void increasePartitions(List<ConfiguredTopic> topics) {
  CreatePartitionsOptions options = dryRun ? new CreatePartitionsOptions().validateOnly(true) :
      new CreatePartitionsOptions();
  Map<String, NewPartitions> newPartitions = topics
      .stream()
      .collect(toMap(ConfiguredTopic::getName, t -> NewPartitions.increaseTo(t.getPartitions())));
  try {
    adminClient.createPartitions(newPartitions, options).all().get();
  } catch (InterruptedException | ExecutionException e) {
    // TODO: FA-10109: Improve exception handling
    throw new RuntimeException(e);
  }
}
 

/** Add topic partitions. */
public Map<String, Object> createTopicPartitions(String clusterAlias, String topic, int totalCount) {
	Map<String, Object> targets = new HashMap<String, Object>();
	int existPartitions = (int) partitionNumbers(clusterAlias, topic);
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaService.getKafkaBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		kafkaService.sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		kafkaService.ssl(prop, clusterAlias);
	}

	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		Map<String, NewPartitions> newPartitions = new HashMap<String, NewPartitions>();
		newPartitions.put(topic, NewPartitions.increaseTo(existPartitions + totalCount));
		adminClient.createPartitions(newPartitions);
		targets.put("status", "success");
		targets.put("info", "Add topic[" + topic + "], before partition[" + existPartitions + "], after partition[" + (existPartitions + totalCount) + "] has successed.");
	} catch (Exception e) {
		LOG.info("Add kafka topic partitions has error, msg is " + e.getMessage());
		e.printStackTrace();
		targets.put("status", "failed");
		targets.put("info", "Add kafka topic partitions has error, msg is " + e.getMessage());
	} finally {
		adminClient.close();
	}
	return targets;
}
 

/**
 * 给topic增加分区
 */
public CreatePartitionsResult addPartitionsForTopic(String topic, int partitions) {
	Map<String, NewPartitions> map = new HashMap<>();
	NewPartitions np = NewPartitions.increaseTo(partitions);
	map.put(topic, np);
	CreatePartitionsOptions cpo = new CreatePartitionsOptions();
	cpo.timeoutMs(5 * 1000);
	return adminClient.createPartitions(map, cpo);
}
 

void maybeAddPartitions(int minPartitionNum) throws ExecutionException, InterruptedException {
  Map<String, KafkaFuture<TopicDescription>> kafkaFutureMap =
      _adminClient.describeTopics(Collections.singleton(_topic)).values();
  KafkaFuture<TopicDescription> topicDescriptions = kafkaFutureMap.get(_topic);
  List<TopicPartitionInfo> partitions = topicDescriptions.get().partitions();

  int partitionNum = partitions.size();
  if (partitionNum < minPartitionNum) {
    LOGGER.info("{} will increase partition of the topic {} in the cluster from {}"
        + " to {}.", this.getClass().toString(), _topic, partitionNum, minPartitionNum);
    Set<Integer> blackListedBrokers = _topicFactory.getBlackListedBrokers(_zkConnect);
    Set<BrokerMetadata> brokers = new HashSet<>();
    for (Node broker : _adminClient.describeCluster().nodes().get()) {
      BrokerMetadata brokerMetadata = new BrokerMetadata(
          broker.id(), null
      );
      brokers.add(brokerMetadata);
    }

    if (!blackListedBrokers.isEmpty()) {
      brokers.removeIf(broker -> blackListedBrokers.contains(broker.id()));
    }

    List<List<Integer>> newPartitionAssignments = newPartitionAssignments(minPartitionNum, partitionNum, brokers, _replicationFactor);

    NewPartitions newPartitions = NewPartitions.increaseTo(minPartitionNum, newPartitionAssignments);

    Map<String, NewPartitions> newPartitionsMap = new HashMap<>();
    newPartitionsMap.put(_topic, newPartitions);
    CreatePartitionsResult createPartitionsResult = _adminClient.createPartitions(newPartitionsMap);

  }
}
 

@Override
public Future<Void> increasePartitions(Topic topic) {
    try {
        String topicName = topic.getTopicName().toString();
        final NewPartitions newPartitions = NewPartitions.increaseTo(topic.getNumPartitions());
        LOGGER.debug("Increasing partitions {}", newPartitions);
        final Map<String, NewPartitions> request = Collections.singletonMap(topicName, newPartitions);
        return mapFuture(adminClient.createPartitions(request).values().get(topicName));
    } catch (Exception e) {
        return Future.failedFuture(e);
    }
}
 

protected CreatePartitionsResult increasePartitionsTo(String topicName, int numPartitions) {
  return adminClient.createPartitions(ImmutableMap.of(topicName, NewPartitions.increaseTo(numPartitions)));
}