kafka.log.LogConfig Java Examples

public static void checkAndCreateTopic(String zkConnect, String topic, int replicas) {
  ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkConnect), IS_SECURE_KAFKA_CLUSTER);

  if (AdminUtils.topicExists(zkUtils, topic)) {
    verifyTopic(zkUtils, topic);
    return;
  }

  int partitions = 1;
  Properties topicConfig = new Properties();
  topicConfig.put(LogConfig.CleanupPolicyProp(), "compact");

  AdminUtils.createTopic(zkUtils, topic, partitions, replicas, topicConfig, RackAwareMode.Enforced$.MODULE$);

  zkClient.close();
  zkUtils.close();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
private static void verifyTopic(ZkUtils zkUtils, String topic) {
  Set topics = new HashSet();
  topics.add(topic);

  // check # partition and the replication factor
  scala.collection.mutable.Map partitionAssignmentForTopics = zkUtils
      .getPartitionAssignmentForTopics(JavaConversions.asScalaSet(topics).toSeq());
  scala.collection.Map partitionAssignment = (scala.collection.Map) partitionAssignmentForTopics.get(topic).get();

  if (partitionAssignment.size() != 1) {
    throw new RuntimeException(String.format("The schema topic %s should have only 1 partition.", topic));
  }

  // check the retention policy
  Properties prop = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic);
  String retentionPolicy = prop.getProperty(LogConfig.CleanupPolicyProp());
  if (retentionPolicy == null || "compact".compareTo(retentionPolicy) != 0) {
    throw new RuntimeException(String.format("The retention policy of the schema topic %s must be compact.", topic));
  }
}
 

protected NewTopic createMetricsTopicFromReporterConfig(CruiseControlMetricsReporterConfig reporterConfig)
    throws CruiseControlMetricsReporterException {
  String cruiseControlMetricsTopic =
      reporterConfig.getString(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_CONFIG);
  Integer cruiseControlMetricsTopicNumPartition =
      reporterConfig.getInt(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_NUM_PARTITIONS_CONFIG);
  Short cruiseControlMetricsTopicReplicaFactor =
      reporterConfig.getShort(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_REPLICATION_FACTOR_CONFIG);
  if (cruiseControlMetricsTopicReplicaFactor <= 0 || cruiseControlMetricsTopicNumPartition <= 0) {
    throw new CruiseControlMetricsReporterException("The topic configuration must explicitly set the replication factor and the num partitions");
  }
  NewTopic newTopic = new NewTopic(cruiseControlMetricsTopic, cruiseControlMetricsTopicNumPartition, cruiseControlMetricsTopicReplicaFactor);

  Map<String, String> config = new HashMap<>(2);
  config.put(LogConfig.RetentionMsProp(),
             Long.toString(reporterConfig.getLong(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_RETENTION_MS_CONFIG)));
  config.put(LogConfig.CleanupPolicyProp(), CRUISE_CONTROL_METRICS_TOPIC_CLEAN_UP_POLICY);
  newTopic.configs(config);
  return newTopic;
}
 

protected void maybeUpdateTopicConfig() {
  try {
    // Retrieve topic config to check and update.
    ConfigResource topicResource = new ConfigResource(ConfigResource.Type.TOPIC, _cruiseControlMetricsTopic);
    DescribeConfigsResult describeConfigsResult = _adminClient.describeConfigs(Collections.singleton(topicResource));
    Config topicConfig = describeConfigsResult.values().get(topicResource).get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    Set<AlterConfigOp> alterConfigOps = new HashSet<>(2);
    Map<String, String> configsToSet = new HashMap<>(2);
    configsToSet.put(LogConfig.RetentionMsProp(), _metricsTopic.configs().get(LogConfig.RetentionMsProp()));
    configsToSet.put(LogConfig.CleanupPolicyProp(), _metricsTopic.configs().get(LogConfig.CleanupPolicyProp()));
    maybeUpdateConfig(alterConfigOps, configsToSet, topicConfig);
    if (!alterConfigOps.isEmpty()) {
      AlterConfigsResult alterConfigsResult = _adminClient.incrementalAlterConfigs(Collections.singletonMap(topicResource, alterConfigOps));
      alterConfigsResult.values().get(topicResource).get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    }
  } catch (InterruptedException | ExecutionException | TimeoutException e) {
    LOG.warn("Unable to update config of Cruise Cruise Control metrics topic {}", _cruiseControlMetricsTopic, e);
  }
}
 

@Test
public void topicAlreadyExists() throws Exception {
  String topic = "topic2";
  int partitions = 1;
  Properties props = new Properties();
  props.setProperty(LogConfig.CleanupPolicyProp(), "compact");
  cluster.createTopic(topic, partitions, 1, props);
  try {
    KafkaStoreUtils.checkAndCreateTopic(cluster.zKConnectString(), topic, REPLICAS);
  } catch (RuntimeException e) {
    fail();
  }
}
 

@Test(expected = RuntimeException.class)
public void topicAlreadyExistsButWrongPartitions() throws Exception {
  String topic = "topic3";
  int partitions = 2;
  Properties props = new Properties();
  props.setProperty(LogConfig.CleanupPolicyProp(), "compact");
  cluster.createTopic(topic, partitions, 1, props);
  KafkaStoreUtils.checkAndCreateTopic(cluster.zKConnectString(), topic, REPLICAS);
}
 

@Test
@SuppressWarnings("unchecked")
public void createTopic_withConfig() throws IOException {
    Properties topicConfig = new Properties();
    topicConfig.setProperty(LogConfig.CleanupPolicyProp(), "delete");

    client.createTopic(topic, 1, 1, topicConfig);

    assertThat(client.getPartitions(), hasItem(new TopicAndPartition(topic, 0)));

    Map<Object, Object> zkTopicConfig = OBJECT_MAPPER.readValue((String) KafkaTests.getZkUtils().zkClient()
            .readData(ZkUtils.getEntityConfigPath(ConfigType.Topic(), topic)), Map.class);
    assertThat(((Map<Object, Object>) zkTopicConfig.get("config")).get(LogConfig.CleanupPolicyProp()), is("delete"));
}
 

@Test
public void getTopicConfig() {
    Properties topicConfig = new Properties();
    topicConfig.setProperty(LogConfig.MinInSyncReplicasProp(), "1");

    client.createTopic(topic, 1, 1, topicConfig);

    assertThat(client.getTopicConfig(topic), is(topicConfig));
}
 

/**
 * Filter out properties from the original config that are not supported by Kafka.
 * For example, we allow users to set replication.factor as a property of the streams
 * and then parse it out so we can pass it separately as Kafka requires. But Kafka
 * will also throw if replication.factor is passed as a property on a new topic.
 *
 * @param originalConfig  The original config to filter
 * @return                The filtered config
 */
private static Map<String, String> filterUnsupportedProperties(Map<String, String> originalConfig) {
  Map<String, String> filteredConfig = new HashMap<>();
  for (Map.Entry<String, String> entry: originalConfig.entrySet()) {
    // Kafka requires replication factor, but not as a property, so we have to filter it out.
    if (!KafkaConfig.TOPIC_REPLICATION_FACTOR().equals(entry.getKey())) {
      if (LogConfig.configNames().contains(entry.getKey())) {
        filteredConfig.put(entry.getKey(), entry.getValue());
      } else {
        LOG.warn("Property '{}' is not a valid Kafka topic config. It will be ignored.", entry.getKey());
      }
    }
  }
  return filteredConfig;
}
 

/**
 * Builds the Kafka topic properties.
 * @return The {@link Properties} of the Kafka Topic.
 * @throws ConfigException - If any of the properties are misconfigured.
 */
public Properties build() throws ConfigException {
    final Properties props = new Properties();

    if(cleanupPolicy.isPresent()) {
        props.setProperty(LogConfig.CleanupPolicyProp(), cleanupPolicy.get());
    }

    LogConfig.validate(props);
    return props;
}
 

@Test
public void validProperty() {
    final Properties props = new KafkaTopicPropertiesBuilder()
        .setCleanupPolicy(LogConfig.Compact())
        .build();

    final Properties expected = new Properties();
    expected.setProperty(LogConfig.CleanupPolicyProp(), LogConfig.Compact());
    assertEquals(expected, props);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  Properties topicConfig = new Properties();
  topicConfig.setProperty(LogConfig.CleanupPolicyProp(), "compact");
  CLUSTER.createTopic(TOPIC, 1, 1, topicConfig);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  Properties topicConfig = new Properties();
  topicConfig.setProperty(LogConfig.CleanupPolicyProp(), "compact");
  CLUSTER.createTopic(TOPIC, 1, 1, topicConfig);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  Properties topicConfig = new Properties();
  topicConfig.setProperty(LogConfig.CleanupPolicyProp(), "compact");
  CLUSTER.createTopic(TOPIC, 1, 1, topicConfig);
}
 

@Test
public void updateTopicConfig() {
    Properties topicConfig = new Properties();
    topicConfig.setProperty(LogConfig.MinInSyncReplicasProp(), "1");

    client.createTopic(topic, 1, 1, topicConfig);

    assertThat(client.getTopicConfig(topic), is(topicConfig));

    topicConfig.setProperty(LogConfig.RetentionMsProp(), "3600000");

    client.updateTopicConfig(topic, topicConfig);

    assertThat(client.getTopicConfig(topic), is(topicConfig));
}