Java Code Examples for kafka.admin.AdminUtils#topicExists()

/**
 * Creates a set of Kafka topics for each topic that does not already exist.
 *
 * @param zookeeperServers - The Zookeeper servers that are used by the Kafka Streams program. (not null)
 * @param topicNames - The topics that will be created. (not null)
 * @param partitions - The number of partitions that each of the topics will have.
 * @param replicationFactor - The replication factor of the topics that are created.
 * @param topicProperties - The optional properties of the topics to create.
 */
public static void createTopics(
        final String zookeeperServers,
        final Set<String> topicNames,
        final int partitions,
        final int replicationFactor,
        final Optional<Properties> topicProperties) {
    requireNonNull(zookeeperServers);
    requireNonNull(topicNames);

    ZkUtils zkUtils = null;
    try {
        zkUtils = ZkUtils.apply(new ZkClient(zookeeperServers, 30000, 30000, ZKStringSerializer$.MODULE$), false);
        for(final String topicName : topicNames) {
            if(!AdminUtils.topicExists(zkUtils, topicName)) {
                AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, topicProperties.orElse(new Properties()), RackAwareMode.Disabled$.MODULE$);
            }
        }
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 

/**
 * Create Kafka topic based on the destination connection string, if it does not already exist.
 * @param connectionString connection string from which to obtain topic name
 * @param numberOfPartitions number of partitions
 * @param topicConfig topic config to use for topic creation
 */
public void createTopic(String connectionString, int numberOfPartitions, Properties topicConfig) {
  Validate.notNull(connectionString, "destination should not be null");
  Validate.notNull(topicConfig, "topicConfig should not be null");
  Validate.isTrue(_zkUtils.isPresent(), "zkUtils should be present");

  String topicName = KafkaTransportProviderUtils.getTopicName(connectionString);
  populateTopicConfig(topicConfig);
  try {
    // Create only if it doesn't exist.
    if (!AdminUtils.topicExists(_zkUtils.get(), topicName)) {
      int replicationFactor = Integer.parseInt(topicConfig.getProperty("replicationFactor", DEFAULT_REPLICATION_FACTOR));
      LOG.info("Creating topic with name {} partitions={} with properties {}", topicName, numberOfPartitions,
              topicConfig);

      AdminUtils.createTopic(_zkUtils.get(), topicName, numberOfPartitions, replicationFactor, topicConfig, RackAwareMode.Disabled$.MODULE$);
    } else {
      LOG.warn("Topic with name {} already exists", topicName);
    }
  } catch (Throwable e) {
    LOG.error("Creating topic {} failed with exception {}", topicName, e);
    throw e;
  }
}
 

public void createTopic(String topic) {
    ZkClient zkClient = new ZkClient(
            kafkaRpcConfig.getZookeeperConnect(),
            kafkaRpcConfig.getSessionTimeout(),
            kafkaRpcConfig.getConnectionTimeout(),
            ZKStringSerializer$.MODULE$);
    try {
        ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(kafkaRpcConfig.getZookeeperConnect()), false);
        Properties topicConfig = kafkaRpcConfig.topicProps();
        if (!AdminUtils.topicExists(zkUtils, topic)) {
            AdminUtils.createTopic(zkUtils, topic, kafkaRpcConfig.getNumPartitions(), 
                    kafkaRpcConfig.getReplicationFactor(), topicConfig, RackAwareMode.Enforced$.MODULE$);
        }
    } finally {
        zkClient.close();
    }
}
 

@Test
public void createTopicInTestEnv() {
	String ZOOKEEPER_CONNECT = "";
	ZkClient zkClient = new ZkClient(new ZkConnection(ZOOKEEPER_CONNECT));
	ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(ZOOKEEPER_CONNECT), false);
	zkClient.setZkSerializer(new ZKStringSerializer());
	int partition = 1;
	int replication = 1;
	String topic = String.format("kafka.test_create_topic_p%s_r%s", partition, replication);
	if (AdminUtils.topicExists(zkUtils, topic)) {
		TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
		System.out.println(topicMetadata);
		AdminUtils.deleteTopic(zkUtils, topic);
	}
	AdminUtils.createTopic(zkUtils, topic, partition, replication, new Properties());
}
 

/**
 * 判断一个topic是否已经存在。 包括已经被标记删除，还没有删除的topic。
 * @param topicName topic的名字
 * @return
 */
public static boolean topicIsExists(String topicName){
    ZkUtils zkUtils = ZkUtils.apply(zkUrl, sessionTimeout, connectionTimeout, JaasUtils.isZkSecurityEnabled());
    boolean topicExists = AdminUtils.topicExists(zkUtils, topicName);
    zkUtils.close();
    return topicExists;
}
 

private void createKafkaTopics() {
    log.info("--------------------------------------------------------------------------");
    log.info("-----------------------Begin to create Kafka topics-----------------------");

    ZkUtils zkUtils = ZkUtils.apply(zkNodes + ":2181", 30000, 30000, JaasUtils.isZkSecurityEnabled());

    if (!AdminUtils.topicExists(zkUtils, "async-task-topic")) {
        AdminUtils.createTopic(zkUtils, "async-task-topic", 1, 1,  new Properties(), new RackAwareMode.Enforced$());
    }

    zkUtils.close();

    log.info("-----------------------Kafka topics created-------------------------------");
    log.info("--------------------------------------------------------------------------");
}
 

@Override
protected void before() throws Throwable {
    

    int zkConnectionTimeout = 6000;
    int zkSessionTimeout = 6000;

    int zookeeperPort = Optional.ofNullable(System.getProperty("zookeeper.port"))
            .filter(s -> !s.isEmpty())
            .map(Integer::parseInt)
            .orElse(ZOOKEEPER_DEFAULT_PORT);
    this.zookeeper = new EmbeddedZookeeperInternal(zookeeperPort);
    this.zkConnect = "127.0.0.1:" + this.zookeeper.getPort();
    this.zookeeperClient = new ZkClient(this.zkConnect, zkSessionTimeout, zkConnectionTimeout,
            ZKStringSerializer$.MODULE$);

    int kafkaPort = Optional.ofNullable(System.getProperty("kafka.port"))
            .filter(s -> !s.isEmpty())
            .map(Integer::parseInt)
            .orElse(KAFKA_DEFAULT_PORT);
    Properties brokerConfigProperties = TestUtils.createBrokerConfig(0, this.zkConnect, this.controlledShutdown,
            true, kafkaPort,
            scala.Option.<SecurityProtocol>apply(null),
            scala.Option.<File>apply(null),
            scala.Option.<Properties>apply(null),
            true, false, 0, false, 0, false, 0, scala.Option.<String>apply(null));
    brokerConfigProperties.setProperty("replica.socket.timeout.ms", "1000");
    brokerConfigProperties.setProperty("controller.socket.timeout.ms", "1000");
    brokerConfigProperties.setProperty("offsets.topic.replication.factor", "1");
    this.kafkaServer = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$);

    ZkUtils zkUtils = new ZkUtils(this.zookeeperClient, null, false);
    Properties properties = new Properties();
    for (String topic : this.topics) {
        if (!AdminUtils.topicExists(zkUtils, topic)) {
            AdminUtils.createTopic(zkUtils, topic, partitions, 1, properties, null);
        }
    }
}
 

private boolean topicExists(String topic) {
  ZkClient zkClient = new ZkClient(cluster.zKConnectString(), SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(cluster.zKConnectString()), IS_SECURE_KAFKA_CLUSTER);
  boolean exists = AdminUtils.topicExists(zkUtils, topic);
  zkClient.close();
  zkUtils.close();
  return exists;
}
 

private void provisionTopic(String topicName,Config config) {
String zooKeeperPropKey = KafkaWriterConfigurationKeys.CLUSTER_ZOOKEEPER;
if(!config.hasPath(zooKeeperPropKey)) {
 log.debug("Topic "+topicName+" is configured without the partition and replication");
 return;
}
String zookeeperConnect = config.getString(zooKeeperPropKey);
int sessionTimeoutMs = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.ZOOKEEPER_SESSION_TIMEOUT, KafkaWriterConfigurationKeys.ZOOKEEPER_SESSION_TIMEOUT_DEFAULT);
int connectionTimeoutMs = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.ZOOKEEPER_CONNECTION_TIMEOUT, KafkaWriterConfigurationKeys.ZOOKEEPER_CONNECTION_TIMEOUT_DEFAULT);
// Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
// createTopic() will only seem to work (it will return without error).  The topic will exist in
// only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
// topic.
ZkClient zkClient = new ZkClient(zookeeperConnect, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
// Security for Kafka was added in Kafka 0.9.0.0
ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect), false);
int partitions = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.PARTITION_COUNT, KafkaWriterConfigurationKeys.PARTITION_COUNT_DEFAULT);
int replication = ConfigUtils.getInt(config, KafkaWriterConfigurationKeys.REPLICATION_COUNT, KafkaWriterConfigurationKeys.PARTITION_COUNT_DEFAULT);
Properties topicConfig = new Properties();
if(AdminUtils.topicExists(zkUtils, topicName)) {
log.debug("Topic"+topicName+" already Exists with replication: "+replication+" and partitions :"+partitions);
   return;
}
try {
   AdminUtils.createTopic(zkUtils, topicName, partitions, replication, topicConfig);
} catch (RuntimeException e) {
   throw new RuntimeException(e);
}
   log.info("Created Topic "+topicName+" with replication: "+replication+" and partitions :"+partitions);
}
 

private void checkAndCreateTopic(String zkConnect, String topic, int partitions, int replicas) {
  ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkConnect), IS_SECURE_KAFKA_CLUSTER);

  if (!AdminUtils.topicExists(zkUtils, topic)) {
    AdminUtils.createTopic(zkUtils, topic, partitions, replicas, new Properties(), RackAwareMode.Enforced$.MODULE$);
  }

  zkUtils.close();
  zkClient.close();
}
 

/**
 * Returns true if topic path exists in zk.
 * Warns user if topic is scheduled for deletion.
 * 
 * @See http://search-hadoop.com/m/4TaT4VWNg8/v=plain
 * */
public boolean topicExits(String topicName) throws Exception {
  int sessionTimeoutMs = 10000;
  int connectionTimeoutMs = 10000;
  ZkClient zkClient = new ZkClient(zkConnects, sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);
  boolean exists = AdminUtils.topicExists(zkClient, topicName);
  if (exists && ZkUtils.pathExists(zkClient, ZkUtils.getDeleteTopicPath(topicName))) {
    System.err.println("Topic "+topicName+" exists but is scheduled for deletion!");
  }
  zkClient.close();
  return exists;
}
 

private void deleteTopic(ZkUtils zookeeperUtils) throws Exception {
  if (AdminUtils.topicExists(zookeeperUtils, topic)) {
    log.info("Deleting topic " + topic + ".");
    AdminUtils.deleteTopic(zookeeperUtils, topic);
  } else {
    log.info("Topic " + topic + " does not exist.");
  }
  while (AdminUtils.topicExists(zookeeperUtils, topic)) {
    Thread.sleep(10);
  }
}
 

@Override
public void createTopicIfNotExist(String topic, int replicationFactor, int partitions) {
    if (!AdminUtils.topicExists(zkClient, topic)) {
        createOrUpdateTopic(topic, replicationFactor, partitions);
    } else {
        logger.info("Topic {} already exists", topic);
    }
}
 

/**
 * Waits for a topic to be created and ready for production by waiting for the topic to be created and then attempting
 * to consume from it once it is created.
 *
 * Note: The topic creation must be issued before this method is called.
 *
 * @param topic the topic to wait for broker assignment
 * @param brokerList the brokers in the Kafka cluster
 * @throws IllegalStateException if the topic is not ready before the timeout ({@value #DEFAULT_TIMEOUT_MS} ms)
 */
public static void waitForTopicCreation(ZkUtils zkUtils, String topic, String brokerList) throws IllegalStateException {
  Validate.notNull(zkUtils);
  Validate.notEmpty(topic);
  Validate.notEmpty(brokerList);

  Duration attemptDuration = Duration.ofMillis(DEFAULT_TIMEOUT_MS);
  Instant expiration = Instant.now().plus(attemptDuration);

  // Wait for topic creation
  while (Instant.now().isBefore(expiration) && !AdminUtils.topicExists(zkUtils, topic)) {
    try {
      Thread.sleep(Duration.ofSeconds(1).toMillis());
    } catch (InterruptedException e) {
      throw new IllegalStateException("Interrupted before topic creation could be verified", e);
    }
  }

  if (Instant.now().isAfter(expiration)) {
    throw new IllegalStateException("Topic was not created within the timeout");
  }

  // Ensure topic is consumable (ready)
  try (KafkaConsumer<byte[], byte[]> consumer = createConsumer(brokerList)) {
    consumer.subscribe(Collections.singleton(topic));
    while (Instant.now().isBefore(expiration)) {
      try {
        consumer.poll(Duration.ofSeconds(1).toMillis());
        return;
      } catch (Exception ignored) {
        // Exception should occur when we are waiting for the broker to be assigned this topic
      }
    }
  }

  throw new IllegalStateException("Topic was not ready within the timeout");
}
 

protected static void createTopic(ZkUtils zkUtils, String topic, int partitionCount) {
  if (!AdminUtils.topicExists(zkUtils, topic)) {
    Properties properties = new Properties();
    properties.put("message.timestamp.type", "LogAppendTime");
    AdminUtils.createTopic(zkUtils, topic, partitionCount, 1, properties, null);
  }
}
 

@VisibleForTesting
protected boolean ifTopicExists(String topicName, ZkUtils zkUtils) {
    return AdminUtils.topicExists(zkUtils, topicName);
}
 

@Override
public boolean topicExists(String topic) throws Exception {
    LOGGER.debug("Checking if topic '" + topic + "' already exists.");
    return AdminUtils.topicExists(zookeeperClient, topic);
}
 

public static boolean topicExists(String zkServers, String topic) {
    try (AutoZkClient zkClient = new AutoZkClient(zkServers)) {
        return AdminUtils.topicExists(zkClient, topic);
    }
}
 

private void createTopic(String topicName, Integer partitions, Integer replication) {
    Boolean isExists =AdminUtils.topicExists(zkClient,topicName);
    if(!isExists){
        AdminUtils.createTopic(zkClient,topicName,partitions,replication,new Properties());
    }
}
 

public boolean topicExists(String name) {
  return AdminUtils.topicExists(zkUtils, name);
}