kafka.zk.AdminZkClient Java Examples

public void createTopic(String topicName, int numPartitions) {
    // setup

    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: CreateTopic " + topicName);
        adminZkClient.createTopic(topicName,numPartitions, 1,new Properties(), RackAwareMode.Disabled$.MODULE$);
    } finally {
        zkClient.close();
    }

}
 

/**
 * @return All topic names
 */
public List<String> listTopics() {
    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: ListTopics ");

            return JavaConversions.asJavaCollection(adminZkClient.getAllTopicConfigs().keys())
                    .stream()
                    .collect(Collectors.toList());

    } finally {
        zkClient.close();
    }
}
 

/**
 * Delete a topic.
 *
 * @param topicName The name of the topic to delete
 */
public void deleteTopic(String topicName) {
    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: DeleteTopic " + topicName);
        adminZkClient.deleteTopic(topicName);
    } finally {
        zkClient.close();
    }
}
 

@Override
public Properties topicConfigs(String topic) {
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(_connectString,
                                                                            ZK_KAFKA_TOPIC_CONFIG_PROVIDER_METRIC_GROUP,
                                                                            ZK_KAFKA_TOPIC_CONFIG_PROVIDER_METRIC_TYPE,
                                                                            _zkSecurityEnabled);
  try {
    AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
    return adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
  } finally {
    KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
  }
}
 

@Override
public Map<String, Properties> allTopicConfigs() {
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(_connectString,
                                                                            ZK_KAFKA_TOPIC_CONFIG_PROVIDER_METRIC_GROUP,
                                                                            ZK_KAFKA_TOPIC_CONFIG_PROVIDER_METRIC_TYPE,
                                                                            _zkSecurityEnabled);
  try {
    AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
    return JavaConversions.mapAsJavaMap(adminZkClient.getAllTopicConfigs());
  } finally {
    KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
  }
}
 

/**
 * Setup the test.
 */
@Before
public void setUp() {
  super.setUp();
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(),
                                                                            "LoadMonitorTaskRunnerGroup",
                                                                            "LoadMonitorTaskRunnerSetup",
                                                                            false);
  AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
  for (int i = 0; i < NUM_TOPICS; i++) {
    adminZkClient.createTopic("topic-" + i, NUM_PARTITIONS, 1, new Properties(), RackAwareMode.Safe$.MODULE$);
  }
  KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
}
 

@Test
void testKafkaServerIsUp(KafkaHelper kafkaHelper) {
    // Setup Zookeeper client
    final String zkConnectionString = kafkaHelper.zookeeperConnectionString();
    final ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zkConnectionString, 2000, 8000, Integer.MAX_VALUE, Time.SYSTEM,"kafka.server", "SessionExpireListener" );
    final KafkaZkClient zkClient = new KafkaZkClient(zooKeeperClient, JaasUtils.isZkSaslEnabled(), Time.SYSTEM);
    final AdminZkClient adminZkClient = new AdminZkClient(zkClient);

    // Create topic
    adminZkClient.createTopic(TOPIC, 1, 1, new Properties(), null);

    // Produce/consume test
    try (KafkaProducer<String, String> producer = kafkaHelper.createStringProducer()) {
        producer.send(new ProducerRecord<>(TOPIC, "keyA", "valueA"));
    }

    try (KafkaConsumer<String, String> consumer = kafkaHelper.createStringConsumer()) {
        consumer.subscribe(Lists.newArrayList(TOPIC));
        ConsumerRecords<String, String> records = consumer.poll(10000);
        Assertions.assertAll(() -> assertThat(records).isNotNull(),
                             () -> assertThat(records.isEmpty()).isFalse());

        ConsumerRecord<String, String> msg = records.iterator().next();
        Assertions.assertAll(() -> assertThat(msg).isNotNull(),
                             () -> assertThat(msg.key()).isEqualTo("keyA"),
                             () -> assertThat(msg.value()).isEqualTo("valueA"));
    }
}
 

@Test
public void testKafkaServerIsUp() {
    // Setup Zookeeper client
    final String zkConnectionString = kafkaRule.helper().zookeeperConnectionString();
    final ZooKeeperClient zooKeeperClient = new ZooKeeperClient(zkConnectionString, 2000, 8000, Integer.MAX_VALUE, Time.SYSTEM,"kafka.server", "SessionExpireListener" );
    final KafkaZkClient zkClient = new KafkaZkClient(zooKeeperClient, JaasUtils.isZkSaslEnabled(), Time.SYSTEM);
    final AdminZkClient adminZkClient = new AdminZkClient(zkClient);

    // Create topic
    adminZkClient.createTopic(TOPIC, 1, 1, new Properties(), null);

    // Produce/consume test
    try (KafkaProducer<String, String> producer = kafkaRule.helper().createStringProducer()) {
        producer.send(new ProducerRecord<>(TOPIC, "keyA", "valueA"));
    }

    try (KafkaConsumer<String, String> consumer = kafkaRule.helper().createStringConsumer()) {
        consumer.subscribe(Lists.newArrayList(TOPIC));
        ConsumerRecords<String, String> records = consumer.poll(TEN_SECONDS);
        assertThat(records).isNotNull();
        assertThat(records.isEmpty()).isFalse();

        ConsumerRecord<String, String> msg = records.iterator().next();
        assertThat(msg).isNotNull();
        assertThat(msg.key()).isEqualTo("keyA");
        assertThat(msg.value()).isEqualTo("valueA");
    }
}
 

@Override
public void deleteTopic(final String topic) throws TopicDeletionException {
    try (KafkaZkClient zkClient = createZkClient()) {
        // this will only trigger topic deletion, but the actual deletion is asynchronous
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        adminZkClient.deleteTopic(topic);
    } catch (final Exception e) {
        throw new TopicDeletionException("Unable to delete topic " + topic, e);
    }
}
 

@Override
public void setRetentionTime(final String topic, final Long retentionMs) throws TopicConfigException {
    try (KafkaZkClient zkClient = createZkClient()) {
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        final Properties topicProps = adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
        topicProps.setProperty("retention.ms", Long.toString(retentionMs));
        adminZkClient.changeTopicConfig(topic, topicProps);
    } catch (final Exception e) {
        throw new TopicConfigException("Unable to update retention time for topic " + topic, e);
    }
}
 

public void createTopic(final String topic, final String zkUrl) {
    try (KafkaZkClient zkClient = createZkClient(zkUrl)) {
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        adminZkClient.createTopic(topic, 1, 1,
                new Properties(), RackAwareMode.Safe$.MODULE$);
    }
}
 

public static String getTopicProperty(final String topic, final String zkPath, final String propertyName) {
    try (KafkaZkClient zkClient = createZkClient(zkPath)) {
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        final Properties topicConfig = adminZkClient.fetchEntityConfig(ConfigType.Topic(), topic);
        return topicConfig.getProperty(propertyName);
    }
}
 

static void createSomeTopics(String zkConnectString) {
	ZooKeeperClient zookeeperClient = new ZooKeeperClient(zkConnectString, 30000, 30000,
			1, Time.SYSTEM, "kafka.server", "SessionExpireListener", Option.empty());
	try (KafkaZkClient kafkaZkClient = new KafkaZkClient(zookeeperClient, false, Time.SYSTEM)) {
		AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
		adminZkClient.createTopic("test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
		adminZkClient.createTopic("dev", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
	}
}
 

ReplicationThrottleHelper(KafkaZkClient kafkaZkClient, Long throttleRate) {
  this._kafkaZkClient = kafkaZkClient;
  this._adminZkClient = new AdminZkClient(kafkaZkClient);
  this._throttleRate = throttleRate;
}
 

@Test
public void testAddingThrottlesWithPreExistingThrottles() throws InterruptedException {
  createTopics();
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(),
      "ReplicationThrottleHelperTestMetricGroup",
      "AddingThrottlesWithNoPreExistingThrottles",
          false);

  final long throttleRate = 100L;

  ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(kafkaZkClient, throttleRate);
  ExecutionProposal proposal = new ExecutionProposal(
      new TopicPartition(TOPIC0, 0),
      100,
      new ReplicaPlacementInfo(0),
      Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)),
      Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));

  ExecutionTask task = completedTaskForProposal(0, proposal);

  // Broker 0 has an existing leader and follower throttle; we expect these to be preserved.
  Properties broker0Config = new Properties();
  long preExistingBroker0ThrottleRate = 200L;
  broker0Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_RATE, String.valueOf(preExistingBroker0ThrottleRate));
  broker0Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_RATE, String.valueOf(preExistingBroker0ThrottleRate));
  ExecutorUtils.changeBrokerConfig(new AdminZkClient(kafkaZkClient), 0, broker0Config);

  // Partition 1 (which is not involved in any execution proposal) has pre-existing throttled
  // replicas (on both leaders and followers); we expect these configurations to be merged
  // with our new throttled replicas.
  Properties topic0Config = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC0);
  topic0Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, "1:0,1:1");
  topic0Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, "1:0,1:1");
  ExecutorUtils.changeTopicConfig(new AdminZkClient(kafkaZkClient), TOPIC0, topic0Config);

  // Topic 1 is not involved in any execution proposal. It has pre-existing throttled replicas.
  Properties topic1Config = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC1);
  topic1Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, "1:1");
  topic1Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, "1:1");
  ExecutorUtils.changeTopicConfig(new AdminZkClient(kafkaZkClient), TOPIC1, topic1Config);

  throttleHelper.setThrottles(Collections.singletonList(proposal));

  assertExpectedThrottledRateForBroker(kafkaZkClient, 0, preExistingBroker0ThrottleRate);
  assertExpectedThrottledRateForBroker(kafkaZkClient, 1, throttleRate);
  assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
  // No throttle on broker 3 because it's not involved in any of the execution proposals:
  assertExpectedThrottledRateForBroker(kafkaZkClient, 3, null);
  // Existing throttled replicas are merged with new throttled replicas for topic 0:
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "0:0,0:1,0:2,1:0,1:1");
  // Existing throttled replicas are unchanged for topic 1:
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, "1:1");

  throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());

  // We expect all throttles related to replica movement to be removed. Specifically,
  // any throttles related to partitions which were not moved will remain.
  // However, we do expect the broker throttles to be removed.
  throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());
  Arrays.asList(0, 1, 2, 3).forEach((i) -> assertExpectedThrottledRateForBroker(kafkaZkClient, i, null));
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "1:0,1:1");
  assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, "1:1");
}