org.apache.kafka.common.errors.TopicExistsException Java Examples

/**
 * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...]
 * @param topic topic to create (if not already existing)
 * @param partitions number of topic partitions
 * @param topicProperties optional topic config properties
 */
public static void maybeCreateTopic(String zkServers,
                                    String topic,
                                    int partitions,
                                    Properties topicProperties) {
  ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false);
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      log.info("No need to create topic {} as it already exists", topic);
    } else {
      log.info("Creating topic {} with {} partition(s)", topic, partitions);
      try {
        AdminUtils.createTopic(
            zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$);
        log.info("Created topic {}", topic);
      } catch (TopicExistsException re) {
        log.info("Topic {} already exists", topic);
      }
    }
  } finally {
    zkUtils.close();
  }
}
 

protected void createCruiseControlMetricsTopic() throws TopicExistsException {
  CruiseControlMetricsUtils.retry(() -> {
    try {
      CreateTopicsResult createTopicsResult = _adminClient.createTopics(Collections.singletonList(_metricsTopic));
      createTopicsResult.values().get(_metricsTopic.name()).get(_metricsTopicAutoCreateTimeoutMs, TimeUnit.MILLISECONDS);
      LOG.info("Cruise Control metrics topic {} is created.", _metricsTopic.name());
      return false;
    } catch (InterruptedException | ExecutionException | TimeoutException e) {
      if (e.getCause() instanceof TopicExistsException) {
        throw (TopicExistsException) e.getCause();
      }
      LOG.warn("Unable to create Cruise Control metrics topic {}.", _metricsTopic.name(), e);
      return true;
    }
  }, _metricsTopicAutoCreateRetries);
}
 

/**
 * Creates a topic in Kafka. If the topic already exists this does nothing.
 * @param topicName the topic name to create.
 * @param partitions the number of partitions to create.
 * @param replicationFactor the number of replicas for the topic.
 */
public void createTopic(final String topicName, final int partitions, final short replicationFactor) {
    // Create admin client
    try (final AdminClient adminClient = getAdminClient()) {
        // Define topic
        final NewTopic newTopic = new NewTopic(topicName, partitions, replicationFactor);

        // Create topic, which is async call.
        final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic));

        // Since the call is Async, Lets wait for it to complete.
        createTopicsResult.values().get(topicName).get();
    } catch (InterruptedException | ExecutionException e) {
        if (!(e.getCause() instanceof TopicExistsException)) {
            throw new RuntimeException(e.getMessage(), e);
        }
        // TopicExistsException - Swallow this exception, just means the topic already exists.
    }
}
 

@PostConstruct
public void init() {
  Map props = new HashMap<>();
  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers);
  props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 50000);
  try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
    try {
      final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
      final CreateTopicsResult createTopicsResult = adminClient
          .createTopics(Collections.singleton(newTopic));
      createTopicsResult.values().get(topic).get();
    } catch (InterruptedException | ExecutionException e) {
      if (e.getCause() instanceof InterruptedException) {
        Thread.currentThread().interrupt();
      }
      if (!(e.getCause() instanceof TopicExistsException)) {
        throw new RuntimeException(e.getMessage(), e);
      }
    }
  }
  LOG.info("Kafka Channel Init");
}
 

@Test(expected = TopicExistsException.class)
public void testSameTopicCannotBeProvisionedAgain() throws Throwable {
	try (AdminClient admin = AdminClient.create(
			Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG,
					embeddedKafka.getEmbeddedKafka().getBrokersAsString()))) {
		admin.createTopics(Collections
				.singletonList(new NewTopic("fooUniqueTopic", 1, (short) 1))).all()
				.get();
		try {
			admin.createTopics(Collections
					.singletonList(new NewTopic("fooUniqueTopic", 1, (short) 1)))
					.all().get();
		}
		catch (Exception ex) {
			assertThat(ex.getCause() instanceof TopicExistsException).isTrue();
			throw ex.getCause();
		}
	}
}
 

private void createTopic(String topicName) throws InterruptedException {
    long start = System.currentTimeMillis();
    while(true) {
        try {
            CLUSTER.createTopic(topicName);
            break;
        } catch (TopicExistsException e) {
            // earlier deleted topic may not yet have been deleted.
            // need to fix in EmbeddedSingleNodeKafkaCluster
            if(System.currentTimeMillis() - start > 30000) {
                throw e;
            }
            Thread.sleep(2000);
        }
    }
}
 

@Override
public void handle(Void v) throws OperatorException {
    kafka.createTopic(topic).onComplete(ar -> {
        if (ar.succeeded()) {
            LOGGER.debug("{}: Created topic '{}' for KafkaTopic '{}'",
                    logContext, topic.getTopicName(), topic.getResourceName());
            handler.handle(ar);
        } else {
            handler.handle(ar);
            if (ar.cause() instanceof TopicExistsException) {
                // TODO reconcile
            } else if (ar.cause() instanceof InvalidReplicationFactorException) {
                // error message is printed in the `reconcile` method
            } else {
                throw new OperatorException(involvedObject, ar.cause());
            }
        }
    });
}
 

@Test
public void create_road_when_topic_exists_works_without_error() throws Exception {
  willThrow(TopicExistsException.class).given(kafkaAdminClient).createTopic(testRoadModel);

  List<PatchOperation> operations = trafficControl.newModel("test_road", testRoadModel);

  verify(kafkaAdminClient).createTopic(testRoadModel);

  assertThat(operations.size(), is(1));
  assertThat(operations.get(0).getOperation(), is(Operation.ADD));
  assertThat(operations.get(0).getPath(), is("/status"));
  assertThat(operations.get(0).getValue(), is(new TrafficControlStatus(true, 6, 1, "")));
}
 

@SuppressWarnings("unchecked")
private CreateTopicsResult createTopicReturningTopicExistsException() {
  CreateTopicsResult createTopicsResult = niceMock(CreateTopicsResult.class);
  expect(createTopicsResult.all())
      .andReturn(failedFuture(new TopicExistsException("Topic already exists")));
  replay(createTopicsResult);
  return createTopicsResult;
}
 

private void createTopic(AdminClient admin) throws CacheInitializationException,
    InterruptedException, ExecutionException, TimeoutException {
    log.info("Creating topic {}", topic);

    int numLiveBrokers = admin.describeCluster().nodes().get(initTimeout, TimeUnit.MILLISECONDS).size();
    if (numLiveBrokers <= 0) {
        throw new CacheInitializationException("No live Kafka brokers");
    }

    int topicReplicationFactor = Math.min(numLiveBrokers, desiredReplicationFactor);
    if (topicReplicationFactor < desiredReplicationFactor) {
        log.warn("Creating the topic "
            + topic
            + " using a replication factor of "
            + topicReplicationFactor
            + ", which is less than the desired one of "
            + desiredReplicationFactor + ". If this is a production environment, it's "
            + "crucial to add more brokers and increase the replication factor of the topic.");
    }

    NewTopic topicRequest = new NewTopic(topic, desiredNumPartitions, (short) topicReplicationFactor);
    topicRequest.configs(
        Collections.singletonMap(
            TopicConfig.CLEANUP_POLICY_CONFIG,
            TopicConfig.CLEANUP_POLICY_COMPACT
        )
    );
    try {
        admin.createTopics(Collections.singleton(topicRequest)).all()
            .get(initTimeout, TimeUnit.MILLISECONDS);
    } catch (ExecutionException e) {
        if (e.getCause() instanceof TopicExistsException) {
            // If topic already exists, ensure that it is configured correctly.
            verifyTopic(admin);
        } else {
            throw e;
        }
    }
}
 

/**
 * Creates the given topic if it does not exist.
 *
 * @param adminClient The adminClient to send createTopics request.
 * @param topicToBeCreated A wrapper around the topic to be created.
 * @return {@code false} if the topic to be created already exists, {@code true} otherwise.
 */
protected static boolean createTopic(AdminClient adminClient, NewTopic topicToBeCreated) {
  try {
    CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singletonList(topicToBeCreated));
    createTopicsResult.values().get(topicToBeCreated.name()).get(CruiseControlMetricsUtils.CLIENT_REQUEST_TIMEOUT_MS,
                                                                 TimeUnit.MILLISECONDS);
    LOG.info("Topic {} has been created.", topicToBeCreated.name());
  } catch (InterruptedException | ExecutionException | TimeoutException e) {
    if (e.getCause() instanceof TopicExistsException) {
      return false;
    }
    throw new IllegalStateException(String.format("Unable to create topic %s.", topicToBeCreated.name()), e);
  }
  return true;
}
 

public static void createTopic(String kafkaTopic, int numOfPartitions, String zkStr, String replicatorFactor) {
  // TopicCommand.main() will call System.exit() finally, which will break maven-surefire-plugin
  try {
    String[] args = new String[]{"--create", "--zookeeper", zkStr, "--replication-factor", replicatorFactor,
        "--partitions", String.valueOf(numOfPartitions), "--topic", kafkaTopic};
    KafkaZkClient zkClient = KafkaZkClient
        .apply(zkStr, false, 3000, 3000, Integer.MAX_VALUE, Time.SYSTEM, "kafka.server",
            "SessionExpireListener");
    TopicCommand.TopicCommandOptions opts = new TopicCommand.TopicCommandOptions(args);
    TopicCommand.createTopic(zkClient, opts);
  } catch (TopicExistsException e) {
    // Catch TopicExistsException otherwise it will break maven-surefire-plugin
    System.out.println("Topic already existed");
  }
}
 

/**
 * Create the topic. This method attempts to create a topic so that all
 * the brokers in the cluster will have partitionToBrokerRatio partitions.  If the topic exists, but has different parameters
 * then this does nothing to update the parameters.
 *
 * TODO: Do we care about rack aware mode?  I would think no because we want to spread the topic over all brokers.
 * @param topic topic name
 * @param replicationFactor the replication factor for the topic
 * @param partitionToBrokerRatio This is multiplied by the number brokers to compute the number of partitions in the topic.
 * @param minPartitionNum partition number to be created at least
 * @param topicConfig additional parameters for the topic for example min.insync.replicas
 * @param adminClient AdminClient object initialized.
 * @return the number of partitions created
 * @throws ExecutionException exception thrown then executing the topic creation fails.
 * @throws InterruptedException exception that's thrown when interrupt occurs.
 */
@SuppressWarnings("unchecked")
public static int createTopicIfNotExists(String topic, short replicationFactor, double partitionToBrokerRatio,
    int minPartitionNum, Properties topicConfig, AdminClient adminClient)
    throws ExecutionException, InterruptedException {
  try {
    if (adminClient.listTopics().names().get().contains(topic)) {
      LOG.info("AdminClient indicates that {} already exists in the cluster. Topic config: {}", topic, topicConfig);
      return getPartitionNumForTopic(adminClient, topic);
    }
    int brokerCount = Utils.getBrokerCount(adminClient);
    int partitionCount = Math.max((int) Math.ceil(brokerCount * partitionToBrokerRatio), minPartitionNum);
    try {
      NewTopic newTopic = new NewTopic(topic, partitionCount, replicationFactor);
      //noinspection rawtypes
      newTopic.configs((Map) topicConfig);

      List<NewTopic> topics = new ArrayList<>();
      topics.add(newTopic);
      CreateTopicsResult result = adminClient.createTopics(topics);

      // waits for this topic creation future to complete, and then returns its result.
      result.values().get(topic).get();
      LOG.info("CreateTopicsResult: {}.", result.values());

    } catch (TopicExistsException e) {
      /* There is a race condition with the consumer. */
      LOG.info("Monitoring topic " + topic + " already exists in the cluster.", e);
      return getPartitionNumForTopic(adminClient, topic);
    }
    LOG.info("Created monitoring topic {} in cluster with {} partitions and replication factor of {}.", topic,
        partitionCount, replicationFactor);

    return partitionCount;
  } finally {
    LOG.info("Completed the topic creation if it doesn't exist for {}.", topic);
  }
}
 

@Override
public void createTopicIfNotExists(Topic topic, Map<String, String> config) {
    NewTopic newTopic = new NewTopic(topic.getName(), topic.getPartitions(), topic.getReplicationFactor());
    newTopic.configs(config);
    try {
        _adminClient.createTopics(Collections.singleton(newTopic)).all().get();
    } catch (ExecutionException | InterruptedException e) {
        if (e.getCause() instanceof TopicExistsException) {
            checkTopicPropertiesMatching(topic);
        } else {
            throw new RuntimeException(e);
        }
    }
}
 

/**
 * 1. operator is notified that a KafkaTopic is created
 * 2. error when creating topic in kafka
 */
@Test
public void testOnKafkaTopicAdded_TopicExistsException(VertxTestContext context) throws InterruptedException {
    Exception createException = new TopicExistsException("");
    resourceAdded(context, createException, null);
    // TODO check a k8s event got created
    // TODO what happens when we subsequently reconcile?
    assertNotReadyStatus(context, createException);
}
 

@Test
public void testKafkaExceptions() {
  //exceptions mapped in KafkaExceptionMapper
  verifyMapperResponse(new BrokerNotAvailableException("some message"), Status.SERVICE_UNAVAILABLE,
      BROKER_NOT_AVAILABLE_ERROR_CODE);

  verifyMapperResponse(new InvalidReplicationFactorException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new SecurityDisabledException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new UnsupportedVersionException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new InvalidPartitionsException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new InvalidRequestException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new UnknownServerException("some message"),Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new UnknownTopicOrPartitionException("some message"), Status.NOT_FOUND,
      KAFKA_UNKNOWN_TOPIC_PARTITION_CODE);
  verifyMapperResponse(new PolicyViolationException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new TopicExistsException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);
  verifyMapperResponse(new InvalidConfigurationException("some message"), Status.BAD_REQUEST,
      KAFKA_BAD_REQUEST_ERROR_CODE);

  //test couple of retriable exceptions
  verifyMapperResponse(new NotCoordinatorException("some message"), Status.INTERNAL_SERVER_ERROR,
      KAFKA_RETRIABLE_ERROR_ERROR_CODE);
  verifyMapperResponse(new NotEnoughReplicasException("some message"), Status.INTERNAL_SERVER_ERROR,
      KAFKA_RETRIABLE_ERROR_ERROR_CODE);

  //test couple of kafka exception
  verifyMapperResponse(new CommitFailedException(), Status.INTERNAL_SERVER_ERROR,
      KAFKA_ERROR_ERROR_CODE);
  verifyMapperResponse(new ConcurrentTransactionsException("some message"), Status.INTERNAL_SERVER_ERROR,
      KAFKA_ERROR_ERROR_CODE);

  //test few general exceptions
  verifyMapperResponse(new NullPointerException("some message"), Status.INTERNAL_SERVER_ERROR,
      Status.INTERNAL_SERVER_ERROR.getStatusCode());
  verifyMapperResponse(new IllegalArgumentException("some message"), Status.INTERNAL_SERVER_ERROR,
      Status.INTERNAL_SERVER_ERROR.getStatusCode());
}
 

@Test(expected = TopicExistsException.class)
public void createTopic_topicExists() {
    client.createTopic(topic, 1, 1);
    client.createTopic(topic, 1, 1);
}
 

public static void main(String[] args) {

        Config config = ConfigFactory.load();

        Properties properties = new Properties();

        properties.put("bootstrap.servers", config.getString("bootstrap.servers"));

        AdminClient client = AdminClient.create(properties);

        HashMap<String, NewTopic> topics = new HashMap<>();

        topics.put(
                config.getString("input.topic.name"),
                new NewTopic(
                        config.getString("input.topic.name"),
                        config.getNumber("input.topic.partitions").intValue(),
                        config.getNumber("input.topic.replication.factor").shortValue())
        );

        topics.put(
                config.getString("output.topic.name"),
                new NewTopic(
                        config.getString("output.topic.name"),
                        config.getNumber("output.topic.partitions").intValue(),
                        config.getNumber("output.topic.replication.factor").shortValue())
        );

        try {
            logger.info("Starting the topics creation");

            CreateTopicsResult result = client.createTopics(topics.values());

            result.values().forEach((topicName, future) -> {
                NewTopic topic = topics.get(topicName);
                future.whenComplete((aVoid, maybeError) ->
                        Optional
                                .ofNullable(maybeError)
                                .map(Try::<Void>failure)
                                .orElse(Try.successful(null))

                                .onFailure(throwable -> logger.error("Topic creation didn't complete:", throwable))
                                .onSuccess((anOtherVoid) -> logger.info(
                                        String.format(
                                                "Topic %s, has been successfully created " +
                                                        "with %s partitions and replicated %s times",
                                                topic.name(),
                                                topic.numPartitions(),
                                                topic.replicationFactor() - 1
                                        )
                                )));
            });

            result.all().get();
        } catch (InterruptedException | ExecutionException e) {
            if (!(e.getCause() instanceof TopicExistsException)) e.printStackTrace();
        } finally {
            client.close();
        }
    }