org.apache.kafka.common.config.TopicConfig Java Examples

public AbstractKafkaRepository(String name, Serde<V> valueSerde, Function<V,String> keyFunction, KafkaAdminService kafkaAdminService, KafkaConfiguration kafkaConfiguration) {
    this.repositoryName = name + "-db";
    this.keyFunction = keyFunction;
    this.producer = KafkaUtils.kafkaProducer(kafkaConfiguration.getBootstrapServers(), StringSerializer.class, JsonNodeSerialializer.class);
    kafkaAdminService.createTopic(kafkaAdminService.buildTopicInfo(repositoryName,TopicConfig.CLEANUP_POLICY_COMPACT));

    Properties props = KafkaUtils.createKStreamProperties(repositoryName + "-stream"+ UUID.randomUUID().toString(), kafkaConfiguration.getBootstrapServers());
    StreamsBuilder builder = new StreamsBuilder();

    final GlobalKTable<String, V> globalKTable = builder.globalTable(repositoryName, materialize(valueSerde));

    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    producer.flush();
    keyValueStore = streams.store(getStoreName(), QueryableStoreTypes.keyValueStore());

    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

}
 

public void createTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = new Properties(defaultTopicConfig);
  topicConfig.setProperty(LEADER_THROTTLED_REPLICAS, WILDCARD);
  topicConfig.setProperty(FOLLOWER_THROTTLED_REPLICAS, WILDCARD);
  RoadType roadType = ofNullable(road.getType()).orElse(RoadType.NORMAL);
  switch (roadType) {
  case NORMAL:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
    break;
  case COMPACT:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);
    break;
  default:
    throw new KafkaException("Unhandled road type \"" + road.getType().name() + "\"");
  }
  AdminUtils
      .createTopic(zkUtils, road.getTopicName(), partitions, replicationFactor, topicConfig, DEFAULT_RACK_AWARE_MODE);
  log.info("Created {} topic {}", roadType, road.getTopicName());
}
 

public void createTopic(String topic, String numPartitions) throws Exception {
    // Create the testAT topic
    Map<String, String> topicProperties = new HashMap<String, String>() {
        {
            put(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1");
        }
    };

    logger.debug("Creating topic: " + topic);
    NewTopic newTopic = new NewTopic(topic, 1, (short) 1).configs(topicProperties);
    adminClient.createTopics(asList(newTopic)).all().get(KAFKA_DEFAULT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
    logger.debug("Topic created.");

    if (numPartitions != null) {
        // create partitions
        logger.debug("Creating: " + numPartitions + " partitions in topic: " + topic);
        Map<String, NewPartitions> partitions = new HashMap<String, NewPartitions>() {
            {
                put (topic, NewPartitions.increaseTo(Integer.parseInt(numPartitions)));
            }
        };

        adminClient.createPartitions(partitions).all().get(KAFKA_DEFAULT_TIMEOUT_MS, TimeUnit.MILLISECONDS);
        logger.debug("Partitions created.");
    }
}
 

/**
 * Fetch stream properties for all intermediate streams.
 *
 * @param config kafka system config
 * @return a {@link Map} from {@code streamId} to stream {@link Properties}
 */
@VisibleForTesting
static Map<String, Properties> getIntermediateStreamProperties(Config config) {
  Map<String, Properties> intermedidateStreamProperties = Collections.emptyMap();
  ApplicationConfig appConfig = new ApplicationConfig(config);

  if (appConfig.getAppMode() == ApplicationConfig.ApplicationMode.BATCH) {
    StreamConfig streamConfig = new StreamConfig(config);
    intermedidateStreamProperties = streamConfig.getStreamIds()
        .stream()
        .filter(streamConfig::getIsIntermediateStream)
        .collect(Collectors.toMap(Function.identity(), streamId -> {
          Properties properties = new Properties();
          properties.putAll(streamConfig.getStreamProperties(streamId));
          properties.putIfAbsent(TopicConfig.RETENTION_MS_CONFIG, String.valueOf(KafkaConfig.DEFAULT_RETENTION_MS_FOR_BATCH()));
          return properties;
        }));
  }

  return intermedidateStreamProperties;
}
 

@Test
public void testMinIsrMoreThanReplicas(VertxTestContext context) {
    KSB ksb = new KSB()
            .addNewTopic("A", false)
                .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2")
                .addNewPartition(0)
                    .replicaOn(0)
                    .leader(0)
                    .isr(0)
                .endPartition()
            .endTopic()
            .addBroker(3);

    KafkaAvailability kafkaAvailability = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaAvailability.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            assertTrue(canRoll,
                    "broker " + brokerId + " should be rollable, being minisr = 2, but only 1 replicas");

            a.flag();
        })));
    }
}
 

@Test
public void testMinIsrEqualsReplicasWithOfflineReplicas(VertxTestContext context) {
    KSB ksb = new KSB()
            .addNewTopic("A", false)
            .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "3")
            .addNewPartition(0)
            .replicaOn(0, 1, 2)
            .leader(0)
            .isr(0, 1)
            .endPartition()
            .endTopic()

            .addBroker(3);

    KafkaAvailability kafkaAvailability = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaAvailability.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            assertTrue(canRoll,
                    "broker " + brokerId + " should be rollable, being minisr = 3, but only 3 replicas");

            a.flag();
        })));
    }
}
 

@Test
public void testMinIsrEqualsReplicas(VertxTestContext context) {
    KSB ksb = new KSB()
            .addNewTopic("A", false)
                .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "3")
                .addNewPartition(0)
                    .replicaOn(0, 1, 2)
                    .leader(0)
                    .isr(0, 1, 2)
                .endPartition()
            .endTopic()

            .addBroker(3);

    KafkaAvailability kafkaAvailability = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaAvailability.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            assertTrue(canRoll,
                    "broker " + brokerId + " should be rollable, being minisr = 3, but only 3 replicas");

            a.flag();
        })));
    }
}
 

/**
 * Retrieve topic minISR config information if it is not cached locally.
 * @param topicsToCheck Set of topics to check.
 */
private void maybeRetrieveAndCacheTopicMinISR(Set<String> topicsToCheck) {
  Set<ConfigResource> topicResourcesToCheck = new HashSet<>(topicsToCheck.size());
  topicsToCheck.stream().filter(t -> !_cachedTopicMinISR.containsKey(t))
                        .forEach(t -> topicResourcesToCheck.add(new ConfigResource(ConfigResource.Type.TOPIC, t)));
  if (topicResourcesToCheck.isEmpty()) {
    return;
  }
  for (Map.Entry<ConfigResource, KafkaFuture<Config>> entry : _adminClient.describeConfigs(topicResourcesToCheck).values().entrySet()) {
    try {
      short topicMinISR = Short.parseShort(entry.getValue().get(DESCRIBE_TOPIC_CONFIG_TIMEOUT_MS, TimeUnit.MILLISECONDS)
                                                .get(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG).value());
      _cachedTopicMinISR.put(entry.getKey().name(), new TopicMinISREntry(topicMinISR, System.currentTimeMillis()));
    } catch (TimeoutException | InterruptedException | ExecutionException e) {
      LOG.warn("Skip attempt to fix replication factor of topic {} due to unable to retrieve its minISR config.",
               entry.getKey().name());
    }
  }
}
 

@Test
public void shouldEnsureCommandTopicHasInfiniteRetention() {
  final Map<String, Object> retentionConfig = ImmutableMap.of(
      TopicConfig.RETENTION_MS_CONFIG, Long.MAX_VALUE
  );
  EasyMock.expect(topicClient.isTopicExists(COMMAND_TOPIC)).andReturn(true);
  EasyMock.expect(topicClient.addTopicConfig(COMMAND_TOPIC, retentionConfig)).andReturn(true);

  EasyMock.replay(topicClient);

  KsqlRestApplication.ensureCommandTopic(restConfig,
                                         topicClient,
                                         COMMAND_TOPIC);

  EasyMock.verify(topicClient);
}
 

@Test
public void shouldRetryAddingTopicConfig() {
  final Map<String, ?> overrides = ImmutableMap.of(
      TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT
  );

  expect(adminClient.describeConfigs(anyObject()))
      .andReturn(topicConfigResponse(
          "peter",
          overriddenConfigEntry(TopicConfig.RETENTION_MS_CONFIG, "12345"),
          defaultConfigEntry(TopicConfig.COMPRESSION_TYPE_CONFIG, "snappy")
      ));

  expect(adminClient.alterConfigs(anyObject()))
      .andReturn(alterTopicConfigResponse(new DisconnectException()))
      .andReturn(alterTopicConfigResponse());
  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  kafkaTopicClient.addTopicConfig("peter", overrides);

  verify(adminClient);
}
 

@Test
public void shouldNotAlterConfigIfConfigNotChanged() {
  final Map<String, ?> overrides = ImmutableMap.of(
      TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT
  );

  expect(adminClient.describeConfigs(topicConfigsRequest("peter")))
      .andReturn(topicConfigResponse(
          "peter",
          overriddenConfigEntry(TopicConfig.CLEANUP_POLICY_CONFIG,
                                TopicConfig.CLEANUP_POLICY_COMPACT)
      ));

  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  kafkaTopicClient.addTopicConfig("peter", overrides);

  verify(adminClient);
}
 

@Test
public void shouldHandleRetriableGetTopicConfigError() {
  expect(adminClient.describeConfigs(anyObject()))
      .andReturn(topicConfigResponse(new DisconnectException()))
      .andReturn(topicConfigResponse(
          "fred",
          overriddenConfigEntry(TopicConfig.RETENTION_MS_CONFIG, "12345"),
          defaultConfigEntry(TopicConfig.COMPRESSION_TYPE_CONFIG, "producer")
      ));
  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  final Map<String, String> config = kafkaTopicClient.getTopicConfig("fred");

  assertThat(config.get(TopicConfig.RETENTION_MS_CONFIG), is("12345"));
  assertThat(config.get(TopicConfig.COMPRESSION_TYPE_CONFIG), is("producer"));
}
 

@Test
public void shouldGetTopicConfig() {
  expect(adminClient.describeConfigs(topicConfigsRequest("fred")))
      .andReturn(topicConfigResponse(
          "fred",
          overriddenConfigEntry(TopicConfig.RETENTION_MS_CONFIG, "12345"),
          defaultConfigEntry(TopicConfig.COMPRESSION_TYPE_CONFIG, "snappy")
      ));
  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  final Map<String, String> config = kafkaTopicClient.getTopicConfig("fred");

  assertThat(config.get(TopicConfig.RETENTION_MS_CONFIG), is("12345"));
  assertThat(config.get(TopicConfig.COMPRESSION_TYPE_CONFIG), is("snappy"));
}
 

@Test
public void shouldCreateTopicWithConfig() {
  // Given:
  final String topicName = UUID.randomUUID().toString();
  final Map<String, String> config = ImmutableMap.of(
      TopicConfig.COMPRESSION_TYPE_CONFIG, "snappy");

  // When:
  client.createTopic(topicName, 2, (short) 1, config);

  // Then:
  assertThatEventually(() -> topicExists(topicName), is(true));
  final TopicDescription topicDescription = getTopicDescription(topicName);
  assertThat(topicDescription.partitions(), hasSize(2));
  assertThat(topicDescription.partitions().get(0).replicas(), hasSize(1));
  final Map<String, String> configs = client.getTopicConfig(topicName);
  assertThat(configs.get(TopicConfig.COMPRESSION_TYPE_CONFIG), is("snappy"));
}
 

private void createTopic(AdminClient adminClient, String topicName, Integer numberOfPartitions, Short replicationFactor) throws InterruptedException, ExecutionException {
    if (!adminClient.listTopics().names().get().contains(topicName)) {
        logger.info("Creating topic {}", topicName);

        final Map<String, String> configs = replicationFactor < 3 ? Map.of(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1") : Map.of();

        final NewTopic newTopic = new NewTopic(topicName, numberOfPartitions, replicationFactor);
        newTopic.configs(configs);
        try {
            CreateTopicsResult topicsCreationResult = adminClient.createTopics(Collections.singleton(newTopic));
            topicsCreationResult.all().get();
        } catch (ExecutionException e) {
            //silent ignore if topic already exists
        }
    }
}
 

public KafkaTopicDetails topicDetails(String topic) {
  Map<Object, List<Object>> map = getPartitionInfo(topic);
  int numPartitions = map.size();
  int numReplicas = map.get(0).size();

  RoadType type;
  Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic);
  String cleanupPolicyConfig = topicConfig
      .getProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
  switch (cleanupPolicyConfig) {
  case TopicConfig.CLEANUP_POLICY_COMPACT:
    type = RoadType.COMPACT;
    break;
  default:
    type = RoadType.NORMAL;
  }
  log.debug("numPartitions: {}, numReplicas: {}", numPartitions, numReplicas);
  return new KafkaTopicDetails(type, numPartitions, numReplicas);
}
 

private void createSinkTopic(
    final String kafkaTopicName,
    final KsqlConfig ksqlConfig,
    final KafkaTopicClient kafkaTopicClient,
    final boolean isCompacted
) {
  int numberOfPartitions =
      (Integer) ksqlConfig.get(KsqlConfig.SINK_NUMBER_OF_PARTITIONS_PROPERTY);
  short numberOfReplications =
      (Short) ksqlConfig.get(KsqlConfig.SINK_NUMBER_OF_REPLICAS_PROPERTY);

  final Map<String, ?> config = isCompacted
      ? ImmutableMap.of(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT)
      : Collections.emptyMap();

  kafkaTopicClient.createTopic(kafkaTopicName,
                               numberOfPartitions,
                               numberOfReplications,
                               config
  );
}
 

@Override
public TopicCleanupPolicy getTopicCleanupPolicy(final String topicName) {
  final String policy = getTopicConfig(topicName)
      .getOrDefault(TopicConfig.CLEANUP_POLICY_CONFIG, "");

  switch (policy) {
    case "compact":
      return TopicCleanupPolicy.COMPACT;
    case "delete":
      return TopicCleanupPolicy.DELETE;
    case "compact+delete":
      return TopicCleanupPolicy.COMPACT_DELETE;
    default:
      throw new KsqlException("Could not get the topic configs for : " + topicName);
  }
}
 

@Test
public void shouldCreateSinkWithCorrectCleanupPolicyNonWindowedTable() {
  KafkaTopicClient topicClientForNonWindowTable = EasyMock.mock(KafkaTopicClient.class);
  KsqlStructuredDataOutputNode outputNode = getKsqlStructuredDataOutputNode(false);
  StreamsBuilder streamsBuilder = new StreamsBuilder();
  Map<String, String> topicConfig = ImmutableMap.of(
      TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);
  topicClientForNonWindowTable.createTopic("output", 4, (short) 3, topicConfig);
  EasyMock.replay(topicClientForNonWindowTable);
  SchemaKStream schemaKStream = outputNode.buildStream(
      streamsBuilder,
      ksqlConfig,
      topicClientForNonWindowTable,
      new FunctionRegistry(),
      new HashMap<>(),
      new MockSchemaRegistryClient());
  assertThat(schemaKStream, instanceOf(SchemaKTable.class));
  EasyMock.verify();

}
 

@PostConstruct
public void init() throws GrokException, IOException {

    GrokCompiler grok = GrokCompiler.newInstance();
    loadFromResource(grok, "/patterns/patterns");
    loadFromResource(grok, "/patterns/firewall");
    loadFromResource(grok, "/patterns/haproxy");
    loadFromResource(grok, "/patterns/java");
    loadFromResource(grok, "/patterns/linux-syslog");
    loadFromResource(grok, "/patterns/nagios");
    loadFromResource(grok, "/patterns/postfix");
    loadFromResource(grok, "/patterns/ruby");

    Map<String, String> patterns = grok.getPatternDefinitions();
    final String topic = "grok-referential-db";
    kafkaAdminService.createTopic(kafkaAdminService.buildTopicInfo(topic, TopicConfig.CLEANUP_POLICY_COMPACT));
    Producer<String, GrokData> grokProducer = kafkaUtils.kafkaGrokProducer();
    for (Map.Entry<String, String> pattern : patterns.entrySet()) {
        log.info(" GrokPatternLoader Produce with key {} value {}",pattern.getKey(),pattern.getValue());
        ProducerRecord<String, GrokData> record = new ProducerRecord<>(topic, pattern.getKey(), GrokData.builder().key(pattern.getKey()).value(pattern.getValue()).build());
        grokProducer.send(record);
    }
}
 

@Test
public void testBelowMinIsr(VertxTestContext context) {
    KSB ksb = new KSB()
        .addNewTopic("A", false)
            .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2")
            .addNewPartition(0)
                .replicaOn(0, 1, 3)
                .leader(0)
                .isr(0, 1)
            .endPartition()
        .endTopic()
        .addNewTopic("B", false)
            .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2")
            .addNewPartition(0)
                .replicaOn(0, 1, 3)
                .leader(1)
                .isr(1)
            .endPartition()
        .endTopic()

        .addBroker(4);

    KafkaAvailability kafkaAvailability = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaAvailability.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            if (brokerId == 4) {
                assertTrue(canRoll,
                        "broker " + brokerId + " should be rollable, having no partitions");
            } else {
                assertFalse(canRoll,
                        "broker " + brokerId + " should not be rollable, being minisr = 2 and it's only replicated on two brokers");
            }
            a.flag();
        })));
    }
}
 

@Test
public void testNoLeader(VertxTestContext context) {
    KSB ksb = new KSB()
            .addNewTopic("A", false)
                .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1")
                .addNewPartition(0)
                    .replicaOn(0, 1, 2)
                    //.leader(0)
                    .isr(1, 2)
                .endPartition()
            .endTopic()
            .addNewTopic("B", false)
                .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1")
                .addNewPartition(0)
                    .replicaOn(0, 1, 2)
                    //.leader(1)
                    .isr(0)
                .endPartition()
            .endTopic()

            .addBroker(3);

    KafkaAvailability kafkaSorted = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaSorted.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            if (brokerId == 0) {
                assertFalse(canRoll,
                        "broker " + brokerId + " should not be rollable, because B/0 would be below min isr");
            } else {
                assertTrue(canRoll,
                        "broker " + brokerId + " should be rollable, being minisr = 1 and having two brokers in its isr");
            }
            a.flag();
        })));
    }
}
 

@Test
public void testAtMinIsr(VertxTestContext context) {
    KSB ksb = new KSB()
        .addNewTopic("A", false)
            .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2")
            .addNewPartition(0)
                .replicaOn(0, 1)
                .leader(0)
                .isr(0, 1)
            .endPartition()
        .endTopic()
        .addNewTopic("B", false)
            .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "2")
            .addNewPartition(0)
                .replicaOn(0, 1)
                .leader(1)
                .isr(0, 1)
            .endPartition()
        .endTopic()

        .addBroker(2);

    KafkaAvailability kafkaAvailability = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaAvailability.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            if (brokerId == 2) {
                assertTrue(canRoll,
                        "broker " + brokerId + " should be rollable, having no partitions");
            } else {
                assertTrue(canRoll,
                        "broker " + brokerId + " should be rollable, because although rolling it will impact availability minisr=|replicas|");
            }
            a.flag();
        })));
    }
}
 

@Test
public void testAboveMinIsr(VertxTestContext context) {
    KSB ksb = new KSB()
            .addNewTopic("A", false)
                .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1")
                .addNewPartition(0)
                    .replicaOn(0, 1, 2)
                    .leader(0)
                    .isr(0, 1, 2)
                .endPartition()
            .endTopic()
            .addNewTopic("B", false)
                .addToConfig(TopicConfig.MIN_IN_SYNC_REPLICAS_CONFIG, "1")
                .addNewPartition(0)
                    .replicaOn(0, 1, 2)
                    .leader(1)
                    .isr(0, 1, 2)
                .endPartition()
            .endTopic()

            .addBroker(3);

    KafkaAvailability kafkaAvailability = new KafkaAvailability(ksb.ac());

    Checkpoint a = context.checkpoint(ksb.brokers.size());
    for (Integer brokerId : ksb.brokers.keySet()) {
        kafkaAvailability.canRoll(brokerId).onComplete(context.succeeding(canRoll -> context.verify(() -> {
            assertTrue(canRoll,
                    "broker " + brokerId + " should be rollable, being minisr = 1 and having two brokers in its isr");
            a.flag();
        })));
    }
}
 

@Provides
@Singleton
@RetryRefTopic
Topic provideRetryRefTopic(MegabusConfiguration megabusConfiguration, KafkaCluster kafkaCluster) {
    kafkaCluster.createTopicIfNotExists(megabusConfiguration.getRetryRefTopic(),
            ImmutableMap.of(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE,
                    TopicConfig.RETENTION_MS_CONFIG, REF_TOPIC_RETENTION,
                    TopicConfig.COMPRESSION_TYPE_CONFIG, COMPRESSION_CONFIGURATION));
    return megabusConfiguration.getRetryRefTopic();
}
 

@Provides
@Singleton
@MissingRefTopic
Topic provideMissingRefTopic(MegabusConfiguration megabusConfiguration, KafkaCluster kafkaCluster) {
    kafkaCluster.createTopicIfNotExists(megabusConfiguration.getMissingRefTopic(),
            ImmutableMap.of(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE,
                    TopicConfig.RETENTION_MS_CONFIG, REF_TOPIC_RETENTION,
                    TopicConfig.COMPRESSION_TYPE_CONFIG, COMPRESSION_CONFIGURATION));
    return megabusConfiguration.getMissingRefTopic();
}
 

@Provides
@Singleton
@MegabusTopic
Topic provideMegabusTopic(MegabusConfiguration megabusConfiguration, KafkaCluster kafkaCluster) {
    kafkaCluster.createTopicIfNotExists(megabusConfiguration.getMegabusTopic(),
            ImmutableMap.of(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT,
                    TopicConfig.DELETE_RETENTION_MS_CONFIG, MEGABUS_DELETE_RETENTION,
                    TopicConfig.COMPRESSION_TYPE_CONFIG, COMPRESSION_CONFIGURATION));
    return megabusConfiguration.getMegabusTopic();
}
 

@Provides
@Singleton
@MegabusRefTopic
Topic provideMegabusRefTopic(MegabusConfiguration megabusConfiguration, KafkaCluster kafkaCluster) {

    kafkaCluster.createTopicIfNotExists(megabusConfiguration.getMegabusRefTopic(),
            ImmutableMap.of(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE,
                    TopicConfig.RETENTION_MS_CONFIG, REF_TOPIC_RETENTION,
                    TopicConfig.COMPRESSION_TYPE_CONFIG, COMPRESSION_CONFIGURATION));
    return megabusConfiguration.getMegabusRefTopic();
}
 

@Test
public void testAlterConfigs(TestContext ctx) {

  KafkaAdminClient adminClient = KafkaAdminClient.create(this.vertx, config);

  ConfigResource resource = new ConfigResource(org.apache.kafka.common.config.ConfigResource.Type.TOPIC, "first-topic");
  // create a entry for updating the retention.ms value on the topic
  ConfigEntry retentionEntry = new ConfigEntry(TopicConfig.RETENTION_MS_CONFIG, "51000");
  Map<ConfigResource, Config> updateConfig = new HashMap<>();
  updateConfig.put(resource, new Config(Collections.singletonList(retentionEntry)));

  adminClient.alterConfigs(updateConfig, ctx.asyncAssertSuccess(v -> {

    adminClient.describeConfigs(Collections.singletonList(resource), ctx.asyncAssertSuccess(describeConfig -> {

      ConfigEntry describeRetentionEntry =
        describeConfig.get(resource)
          .getEntries()
          .stream()
          .filter(entry -> entry.getName().equals(TopicConfig.RETENTION_MS_CONFIG))
          .collect(Collectors.toList())
          .get(0);

      ctx.assertEquals("51000", describeRetentionEntry.getValue());
      adminClient.close();
    }));
  }));
}
 

/**
 * Example about altering resources configuration like topic or broker
 * @param adminClient Kafka admin client instance
 */
public void exampleAlterConfigs(KafkaAdminClient adminClient) {
  ConfigResource resource = new ConfigResource(org.apache.kafka.common.config.ConfigResource.Type.TOPIC, "my-topic");
  // create a entry for updating the retention.ms value on the topic
  ConfigEntry retentionEntry = new ConfigEntry(TopicConfig.RETENTION_MS_CONFIG, "51000");
  Map<ConfigResource, Config> updateConfig = new HashMap<>();
  updateConfig.put(resource, new Config(Collections.singletonList(retentionEntry)));
  adminClient.alterConfigs(updateConfig)
    .onSuccess(v -> {
      // configuration altered successfully
    })
    .onFailure(cause -> {
      // something went wrong when altering configs
    });
}