org.apache.kafka.clients.admin.KafkaAdminClient Java Examples

public static void describeTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        // Make async call to describe the topic.
        final DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singleton(topicName));

        TopicDescription description = describeTopicsResult.values().get(topicName).get();
        System.out.print(description.toString());
    } catch (Exception e) {
        System.out.print("Describe denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 

public void init(Properties props) {
  port = (int) props.get(PORT);
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  serverStartable = new KafkaServerStartable(new KafkaConfig(props));
  final Map<String, Object> config = new HashMap<>();
  config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
  config.put(AdminClientConfig.CLIENT_ID_CONFIG, "Kafka2AdminClient-" + UUID.randomUUID().toString());
  config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, 15000);
  adminClient = KafkaAdminClient.create(config);
}
 

@Test
public void testCreateTopic() throws Exception {
        final String topic = "test";
        Properties properties = new Properties();
        properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
        properties.put("client.id", "test-consumer-id");
        properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
        AdminClient client = KafkaAdminClient.create(properties);
        CreateTopicsResult result = client.createTopics(Arrays.asList(new NewTopic(topic, 1, (short) 1)));
        result.values().get(topic).get();
        for (Map.Entry<String, KafkaFuture<Void>> entry : result.values().entrySet()) {
            System.out.println("Create Topic : " + entry.getKey() + " " +
                    "isCancelled : " + entry.getValue().isCancelled() + " " +
                    "isCompletedExceptionally : " + entry.getValue().isCompletedExceptionally() + " " +
                    "isDone : " + entry.getValue().isDone());
        }
}
 

public static void createTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        int numPartitions = 8;
        short replicationFactor = (short)3;
        final NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);

        final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic));
        createTopicsResult.values().get(topicName).get();
        System.out.print("Topic " + topicName + " created");
    } catch (Exception e) {
        System.out.print("Create Topics denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 

public static void describeTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        // Make async call to describe the topic.
        final DescribeTopicsResult describeTopicsResult = adminClient.describeTopics(Collections.singleton(topicName));

        TopicDescription description = describeTopicsResult.values().get(topicName).get();
        System.out.print(description.toString());
    } catch (Exception e) {
        System.out.print("Describe denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 

public static void createTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        int numPartitions = 8;
        short replicationFactor = (short)3;
        final NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);

        final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic));
        createTopicsResult.values().get(topicName).get();
        System.out.print("Topic " + topicName + " created");
    } catch (Exception e) {
        System.out.print("Create Topics denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 

@PostConstruct
public void init() {
  Map props = new HashMap<>();
  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers);
  props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 50000);
  try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
    try {
      final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
      final CreateTopicsResult createTopicsResult = adminClient
          .createTopics(Collections.singleton(newTopic));
      createTopicsResult.values().get(topic).get();
    } catch (InterruptedException | ExecutionException e) {
      if (e.getCause() instanceof InterruptedException) {
        Thread.currentThread().interrupt();
      }
      if (!(e.getCause() instanceof TopicExistsException)) {
        throw new RuntimeException(e.getMessage(), e);
      }
    }
  }
  LOG.info("Kafka Channel Init");
}
 

private Set<String> listTopicsOrThrowException() throws ExecutionException, InterruptedException {
    // Use a key store helper if a self signed certificate is provided
    KeyStoreHelper brokerKeyStoreHelper = certificate != null ? new KeyStoreHelper(certificate, "brokerCertificate").store() : null;
    Properties properties = getKafkaAdminClientConfiguration(brokers, transportProtocol, brokerKeyStoreHelper);
    try (AdminClient client = KafkaAdminClient.create(properties)) {
        ListTopicsResult topics = client.listTopics();
        return topics.names().get();
    } finally {
        if (brokerKeyStoreHelper != null) {
            // Clean up temporary resources used by key store
            boolean keystoreDeleted = brokerKeyStoreHelper.clean();
            if (!keystoreDeleted) {
                LOG.warn("Impossible to delete temporary keystore located at " + brokerKeyStoreHelper.getKeyStorePath());
            }
        }
    }
}
 

boolean getRunningBrokers(Properties connectionProperties ) {

            boolean found = false;
            try {
                AdminClient adminClient = KafkaAdminClient.create(connectionProperties);
                DescribeClusterResult describeClusterResult = adminClient.describeCluster();
                Collection<Node> brokers = describeClusterResult.nodes().get();
                if (!brokers.isEmpty()) {
                    found = true;
                }
            } catch (Exception e) {
                //gulp down any exceptions, the waiting method will control any audit logging
                //but keep a copy for reference
                lastException = e;
            }

            return found;
        }
 

/**
 * Create a new AdminClient instance.
 * @param clusterConfig What cluster to connect to.
 * @param clientId What clientId to associate the connection with.
 * @return AdminClient instance.
 */
public AdminClient create(final ClusterConfig clusterConfig, final String clientId) {
    // Create a map
    final Map<String, Object> config = configUtil.applyCommonSettings(clusterConfig, clientId);

    // Build admin client.
    return KafkaAdminClient.create(config);
}
 

@Override
public int run() {
  try (AdminClient kafka = KafkaAdminClient.create(kafkaConfig())) {
    TopicService topicService = new TopicServiceImpl(kafka, true);
    Collection<ConfiguredTopic> existing = topicService.listExisting(true).values()
        .stream()
        .sorted(Comparator.comparing(ConfiguredTopic::getName))
        .collect(Collectors.toList());
    System.out.println(MAPPER.writerWithDefaultPrettyPrinter().writeValueAsString(existing));
    return SUCCESS;
  } catch (JsonProcessingException e) {
    LOG.error("Failed to dump config", e);
    return FAILURE;
  }
}
 

public static void deleteTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        final DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(Collections.singleton(topicName));
        deleteTopicsResult.values().get(topicName).get();
        System.out.print("Topic " + topicName + " deleted");
    } catch (Exception e) {
        System.out.print("Delete Topics denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 

public SimpleKafkaStreams() throws InterruptedException, ExecutionException {
    properties = buildProperties(defaultProps, System.getenv(), KAFKA_ENV_PREFIX);
    inputTopic = System.getenv().getOrDefault("INPUT_TOPIC","sample");
    outputTopic = System.getenv().getOrDefault("OUTPUT_TOPIC","output");

    final Integer numberOfPartitions =  Integer.valueOf(System.getenv().getOrDefault("NUMBER_OF_PARTITIONS","2"));
    final Short replicationFactor =  Short.valueOf(System.getenv().getOrDefault("REPLICATION_FACTOR","3"));

    AdminClient adminClient = KafkaAdminClient.create(properties);
    createTopic(adminClient, inputTopic, numberOfPartitions, replicationFactor);
    createTopic(adminClient, outputTopic, numberOfPartitions, replicationFactor);

}
 

public static void deleteTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        final DeleteTopicsResult deleteTopicsResult = adminClient.deleteTopics(Collections.singleton(topicName));
        deleteTopicsResult.values().get(topicName).get();
        System.out.print("Topic " + topicName + " deleted");
    } catch (Exception e) {
        System.out.print("Delete Topics denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 

public SimpleProducer() throws ExecutionException, InterruptedException {
    properties = buildProperties(defaultProps, System.getenv(), KAFKA_ENV_PREFIX);
    topicName = System.getenv().getOrDefault("TOPIC","sample");
    messageBackOff = Long.valueOf(System.getenv().getOrDefault("MESSAGE_BACKOFF","100"));

    final Integer numberOfPartitions =  Integer.valueOf(System.getenv().getOrDefault("NUMBER_OF_PARTITIONS","2"));
    final Short replicationFactor =  Short.valueOf(System.getenv().getOrDefault("REPLICATION_FACTOR","3"));

    AdminClient adminClient = KafkaAdminClient.create(properties);
    createTopic(adminClient, topicName, numberOfPartitions, replicationFactor);
}
 

@Override
public void init(SecorConfig config) {
    mZookeeperConnector = new ZookeeperConnector(config);
    mPollTimeout = config.getNewConsumerPollTimeoutSeconds();
    Properties props = new Properties();
    props.put("bootstrap.servers", config.getKafkaSeedBrokerHost() + ":" + config.getKafkaSeedBrokerPort());
    props.put("enable.auto.commit", false);
    props.put("auto.offset.reset", "earliest");
    props.put("key.deserializer", ByteArrayDeserializer.class);
    props.put("value.deserializer", ByteArrayDeserializer.class);
    props.put("max.poll.records", 1);
    mKafkaConsumer = new KafkaConsumer<>(props);
    mKafkaAdminClient = KafkaAdminClient.create(props);
}
 

public void createConnection(String brokersUrl) {
    kafkaConnectionProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokersUrl);
    kafkaProducerProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokersUrl);
    kafkaConsumerProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokersUrl);

    logger.debug("Creating non-secured Kafka connection: " + brokersUrl);
    kafkaConnectionProperties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT");
    kafkaProducerProperties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT");
    kafkaConsumerProperties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "PLAINTEXT");

    adminClient = KafkaAdminClient.create(kafkaConnectionProperties);
    logger.debug("Kafka connection created.");
}
 

public void createConnection(String brokersUrl, String keystore, String keypass, String truststore, String trustpass) throws InterruptedException {
    if (adminClient != null) {
        closeConnection();
    }

    kafkaConnectionProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokersUrl);
    kafkaProducerProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokersUrl);
    kafkaConsumerProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, brokersUrl);

    logger.debug("Creating secured Kafka connection: " + brokersUrl);
    kafkaConnectionProperties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL");
    kafkaConnectionProperties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststore);
    kafkaConnectionProperties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, trustpass);
    kafkaConnectionProperties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keystore);
    kafkaConnectionProperties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, keypass);

    kafkaProducerProperties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL");
    kafkaProducerProperties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststore);
    kafkaProducerProperties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, trustpass);
    kafkaProducerProperties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keystore);
    kafkaProducerProperties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, keypass);

    kafkaConsumerProperties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SSL");
    kafkaConsumerProperties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, truststore);
    kafkaConsumerProperties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, trustpass);
    kafkaConsumerProperties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, keystore);
    kafkaConsumerProperties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, keypass);

    adminClient = KafkaAdminClient.create(kafkaConnectionProperties);
    logger.debug("Kafka connection created.");
}
 

@Override
public int run() {
  try (AdminClient kafka = KafkaAdminClient.create(kafkaConfig())) {
    TopicService topicService = new TopicServiceImpl(kafka, dryrun);
    Enforcer enforcer = new Enforcer(topicService, configuredTopics(), !unsafemode);
    return doRun(enforcer, () -> false);
  }
}
 

@Before
public void setup() {
  adminClient = mock(KafkaAdminClient.class);
}
 

/**
 * Creates a Kafka AdminClient connected to our test server.
 * @return Kafka AdminClient instance.
 */
public AdminClient getAdminClient() {
    return KafkaAdminClient.create(buildDefaultClientConfig());
}