org.apache.kafka.clients.admin.NewTopic Java Examples

The following examples show how to use org.apache.kafka.clients.admin.NewTopic. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TopologyBuilderAdminClient.java    From kafka-topology-builder with MIT License 7 votes vote down vote up
public void createTopic(Topic topic, String fullTopicName) throws IOException {

    int numPartitions =
        Integer.parseInt(topic.getConfig().getOrDefault(TopicManager.NUM_PARTITIONS, "3"));
    short replicationFactor =
        Short.parseShort(topic.getConfig().getOrDefault(TopicManager.REPLICATION_FACTOR, "2"));

    NewTopic newTopic =
        new NewTopic(fullTopicName, numPartitions, replicationFactor).configs(topic.rawConfig());
    Collection<NewTopic> newTopics = Collections.singleton(newTopic);
    try {
      createAllTopics(newTopics);
    } catch (ExecutionException | InterruptedException e) {
      LOGGER.error(e);
      throw new IOException(e);
    }
  }
 
Example #2
Source File: KafkaTestContainerManager.java    From apicurio-registry with Apache License 2.0 7 votes vote down vote up
private void createTopics(String bootstrapServers) {
    Properties properties = new Properties();
    properties.put("bootstrap.servers", bootstrapServers);
    properties.put("connections.max.idle.ms", 10000);
    properties.put("request.timeout.ms", 5000);
    try (AdminClient client = AdminClient.create(properties)) {
        CreateTopicsResult result = client.createTopics(Arrays.asList(
                new NewTopic("storage-topic", 1, (short) 1),
                new NewTopic("global-id-topic", 1, (short) 1),
                new NewTopic("snapshot-topic", 1, (short) 1)
        ));
        try {
            result.all().get();
        } catch ( InterruptedException | ExecutionException e ) {
            throw new IllegalStateException(e);
        }
    }
}
 
Example #3
Source File: NamingChangelogAndRepartitionTopics.java    From kafka-tutorials with Apache License 2.0 7 votes vote down vote up
public void createTopics(final Properties envProps) {
  final Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  try (final AdminClient client = AdminClient.create(config)) {

    final List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
        envProps.getProperty("input.topic.name"),
        Integer.parseInt(envProps.getProperty("input.topic.partitions")),
        Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

    topics.add(new NewTopic(
        envProps.getProperty("output.topic.name"),
        Integer.parseInt(envProps.getProperty("output.topic.partitions")),
        Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    topics.add(new NewTopic(
        envProps.getProperty("join.topic.name"),
        Integer.parseInt(envProps.getProperty("join.topic.partitions")),
        Short.parseShort(envProps.getProperty("join.topic.replication.factor"))));

    client.createTopics(topics);
  }
}
 
Example #4
Source File: CruiseControlMetricsReporter.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
protected NewTopic createMetricsTopicFromReporterConfig(CruiseControlMetricsReporterConfig reporterConfig)
    throws CruiseControlMetricsReporterException {
  String cruiseControlMetricsTopic =
      reporterConfig.getString(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_CONFIG);
  Integer cruiseControlMetricsTopicNumPartition =
      reporterConfig.getInt(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_NUM_PARTITIONS_CONFIG);
  Short cruiseControlMetricsTopicReplicaFactor =
      reporterConfig.getShort(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_REPLICATION_FACTOR_CONFIG);
  if (cruiseControlMetricsTopicReplicaFactor <= 0 || cruiseControlMetricsTopicNumPartition <= 0) {
    throw new CruiseControlMetricsReporterException("The topic configuration must explicitly set the replication factor and the num partitions");
  }
  NewTopic newTopic = new NewTopic(cruiseControlMetricsTopic, cruiseControlMetricsTopicNumPartition, cruiseControlMetricsTopicReplicaFactor);

  Map<String, String> config = new HashMap<>(2);
  config.put(LogConfig.RetentionMsProp(),
             Long.toString(reporterConfig.getLong(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_RETENTION_MS_CONFIG)));
  config.put(LogConfig.CleanupPolicyProp(), CRUISE_CONTROL_METRICS_TOPIC_CLEAN_UP_POLICY);
  newTopic.configs(config);
  return newTopic;
}
 
Example #5
Source File: RunningAverage.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
/**
 * Create topics using AdminClient API
 */
private void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();

  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();

  topics.add(new NewTopic(
      envProps.getProperty("input.ratings.topic.name"),
      parseInt(envProps.getProperty("input.ratings.topic.partitions")),
      parseShort(envProps.getProperty("input.ratings.topic.replication.factor"))));

  topics.add(new NewTopic(
      envProps.getProperty("output.rating-averages.topic.name"),
      parseInt(envProps.getProperty("output.rating-averages.topic.partitions")),
      parseShort(envProps.getProperty("output.rating-averages.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();

}
 
Example #6
Source File: FindDistinctEvents.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    topics.add(new NewTopic(
            envProps.getProperty("input.topic.name"),
            Integer.parseInt(envProps.getProperty("input.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 
Example #7
Source File: AdminClientWrapper.java    From hdinsight-kafka-java-get-started with MIT License 6 votes vote down vote up
public static void createTopics(String brokers, String topicName) throws IOException {
    // Set properties used to configure admin client
    Properties properties = getProperties(brokers);

    try (final AdminClient adminClient = KafkaAdminClient.create(properties)) {
        int numPartitions = 8;
        short replicationFactor = (short)3;
        final NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);

        final CreateTopicsResult createTopicsResult = adminClient.createTopics(Collections.singleton(newTopic));
        createTopicsResult.values().get(topicName).get();
        System.out.print("Topic " + topicName + " created");
    } catch (Exception e) {
        System.out.print("Create Topics denied\n");
        System.out.print(e.getMessage());
        //throw new RuntimeException(e.getMessage(), e);
    }
}
 
Example #8
Source File: KafkaChannelAutoConfiguration.java    From servicecomb-pack with Apache License 2.0 6 votes vote down vote up
@PostConstruct
public void init() {
  Map props = new HashMap<>();
  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrap_servers);
  props.put(ProducerConfig.MAX_BLOCK_MS_CONFIG, 50000);
  try (final AdminClient adminClient = KafkaAdminClient.create(props)) {
    try {
      final NewTopic newTopic = new NewTopic(topic, numPartitions, replicationFactor);
      final CreateTopicsResult createTopicsResult = adminClient
          .createTopics(Collections.singleton(newTopic));
      createTopicsResult.values().get(topic).get();
    } catch (InterruptedException | ExecutionException e) {
      if (e.getCause() instanceof InterruptedException) {
        Thread.currentThread().interrupt();
      }
      if (!(e.getCause() instanceof TopicExistsException)) {
        throw new RuntimeException(e.getMessage(), e);
      }
    }
  }
  LOG.info("Kafka Channel Init");
}
 
Example #9
Source File: TopicServiceImplTest.java    From kafka-helmsman with MIT License 6 votes vote down vote up
@Test
public void testCreate() {
  TopicService service = new TopicServiceImpl(adminClient, true);
  CreateTopicsResult createTopicsResult = mock(CreateTopicsResult.class);
  when(createTopicsResult.all()).thenReturn(KafkaFuture.completedFuture(null));
  when(adminClient.createTopics(any(Collection.class),
      any(CreateTopicsOptions.class))).thenReturn(createTopicsResult);

  service.create(Collections.singletonList(
      new ConfiguredTopic("test", 1, (short) 2, Collections.emptyMap())));

  ArgumentCaptor<List> newTopics = ArgumentCaptor.forClass(List.class);
  ArgumentCaptor<CreateTopicsOptions> options = ArgumentCaptor.forClass(CreateTopicsOptions.class);
  verify(adminClient).createTopics((Collection<NewTopic>) newTopics.capture(), options.capture());
  Assert.assertEquals(1, newTopics.getValue().size());
  Assert.assertEquals("test", ((NewTopic) newTopics.getValue().get(0)).name());
  Assert.assertEquals(2, ((NewTopic) newTopics.getValue().get(0)).replicationFactor());
  Assert.assertTrue(options.getValue().shouldValidateOnly());
}
 
Example #10
Source File: TopicSerializationTest.java    From strimzi-kafka-operator with Apache License 2.0 6 votes vote down vote up
@Test
public void testToNewTopic() {
    Topic topic = new Topic.Builder()
            .withTopicName("test-topic")
            .withConfigEntry("foo", "bar")
            .withNumPartitions(3)
            .withNumReplicas((short) 2)
            .withMapName("gee")
            .build();
    NewTopic newTopic = TopicSerialization.toNewTopic(topic, null);
    assertThat(newTopic.name(), is("test-topic"));
    assertThat(newTopic.numPartitions(), is(3));
    assertThat(newTopic.replicationFactor(), is((short) 2));
    assertThat(newTopic.replicasAssignments(), is(nullValue()));
    assertThat(newTopic.configs(), is(singletonMap("foo", "bar")));
}
 
Example #11
Source File: TransformEvents.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {

        Map<String, Object> config = new HashMap<>();
        config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));

        try (AdminClient adminClient = AdminClient.create(config)) {

            List<NewTopic> topics = new ArrayList<>();
            topics.add(new NewTopic(
                    envProps.getProperty("input.topic.name"),
                    Integer.parseInt(envProps.getProperty("input.topic.partitions")),
                    Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
            topics.add(new NewTopic(
                    envProps.getProperty("output.topic.name"),
                    Integer.parseInt(envProps.getProperty("output.topic.partitions")),
                    Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));
    
            adminClient.createTopics(topics);

        }

    }
 
Example #12
Source File: StreamsIngest.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 
Example #13
Source File: FkJoinTableToTable.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(final Properties envProps) {
    final Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    final AdminClient client = AdminClient.create(config);

    final List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("album.topic.name"),
            Integer.parseInt(envProps.getProperty("album.topic.partitions")),
            Short.parseShort(envProps.getProperty("album.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("tracks.purchase.topic.name"),
            Integer.parseInt(envProps.getProperty("tracks.purchase.topic.partitions")),
            Short.parseShort(envProps.getProperty("tracks.purchase.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("music.interest.topic.name"),
            Integer.parseInt(envProps.getProperty("music.interest.topic.partitions")),
            Short.parseShort(envProps.getProperty("music.interest.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 
Example #14
Source File: AggregatingSum.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 
Example #15
Source File: TumblingWindow.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();
    Map<String, String> topicConfigs = new HashMap<>();
    topicConfigs.put("retention.ms", Long.toString(Long.MAX_VALUE));

    NewTopic ratings = new NewTopic(envProps.getProperty("rating.topic.name"),
                                    Integer.parseInt(envProps.getProperty("rating.topic.partitions")),
                                    Short.parseShort(envProps.getProperty("rating.topic.replication.factor")));
    ratings.configs(topicConfigs);
    topics.add(ratings);

    NewTopic counts = new NewTopic(envProps.getProperty("rating.count.topic.name"),
                                   Integer.parseInt(envProps.getProperty("rating.count.topic.partitions")),
                                   Short.parseShort(envProps.getProperty("rating.count.topic.replication.factor")));
    counts.configs(topicConfigs);
    topics.add(counts);


    client.createTopics(topics);
    client.close();
}
 
Example #16
Source File: AggregatingCount.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 
Example #17
Source File: TransformStream.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("input.topic.name"),
            Integer.parseInt(envProps.getProperty("input.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 
Example #18
Source File: MergeStreams.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("input.rock.topic.name"),
            Integer.parseInt(envProps.getProperty("input.rock.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.rock.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("input.classical.topic.name"),
            Integer.parseInt(envProps.getProperty("input.classical.topic.partitions")),
            Short.parseShort(envProps.getProperty("input.classical.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("output.topic.name"),
            Integer.parseInt(envProps.getProperty("output.topic.partitions")),
            Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 
Example #19
Source File: SamplingUtilsTest.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testMaybeIncreasePartitionCount() throws InterruptedException, ExecutionException, TimeoutException {
  AdminClient adminClient = EasyMock.createMock(AdminClient.class);
  NewTopic topicToAddPartitions = SamplingUtils.wrapTopic(MOCK_TOPIC, MOCK_DESIRED_PARTITION_COUNT,
                                                          MOCK_REPLICATION_FACTOR, MOCK_DESIRED_RETENTION_MS);
  DescribeTopicsResult describeTopicsResult = EasyMock.createMock(DescribeTopicsResult.class);
  KafkaFuture<TopicDescription> topicDescriptionFuture = EasyMock.createMock(KafkaFuture.class);
  TopicDescription topicDescription = EasyMock.createMock(TopicDescription.class);
  Map<String, KafkaFuture<TopicDescription>> describeTopicsValues = Collections.singletonMap(MOCK_TOPIC, topicDescriptionFuture);
  Map<String, KafkaFuture<Void>> createPartitionsValues = Collections.singletonMap(MOCK_TOPIC, EasyMock.createMock(KafkaFuture.class));
  CreatePartitionsResult createPartitionsResult = EasyMock.createMock(CreatePartitionsResult.class);

  EasyMock.expect(adminClient.describeTopics(Collections.singletonList(MOCK_TOPIC))).andReturn(describeTopicsResult);
  EasyMock.expect(describeTopicsResult.values()).andReturn(describeTopicsValues);
  EasyMock.expect(topicDescriptionFuture.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(topicDescription);
  EasyMock.expect(topicDescription.partitions()).andReturn(MOCK_PARTITIONS);
  EasyMock.expect(adminClient.createPartitions(Collections.singletonMap(MOCK_TOPIC, EasyMock.anyObject())))
          .andReturn(createPartitionsResult);
  EasyMock.expect(createPartitionsResult.values()).andReturn(createPartitionsValues);

  EasyMock.replay(adminClient, describeTopicsResult, topicDescriptionFuture, topicDescription, createPartitionsResult);
  boolean increasePartitionCount = SamplingUtils.maybeIncreasePartitionCount(adminClient, topicToAddPartitions);

  EasyMock.verify(adminClient, describeTopicsResult, topicDescriptionFuture, topicDescription, createPartitionsResult);
  assertTrue(increasePartitionCount);
}
 
Example #20
Source File: DynamicOutputTopic.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(final Properties envProps) {
    final Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    try (final AdminClient client = AdminClient.create(config)) {

    final List<NewTopic> topics = new ArrayList<>();

        topics.add(new NewTopic(
                envProps.getProperty("input.topic.name"),
                Integer.parseInt(envProps.getProperty("input.topic.partitions")),
                Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));

        topics.add(new NewTopic(
                envProps.getProperty("output.topic.name"),
                Integer.parseInt(envProps.getProperty("output.topic.partitions")),
                Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

        topics.add(new NewTopic(
            envProps.getProperty("special.order.topic.name"),
            Integer.parseInt(envProps.getProperty("special.order.topic.partitions")),
            Short.parseShort(envProps.getProperty("special.order.topic.replication.factor"))));

        client.createTopics(topics);
    }
}
 
Example #21
Source File: FilterEvents.java    From kafka-tutorials with Apache License 2.0 6 votes vote down vote up
public void createTopics(Properties envProps) {
  Map<String, Object> config = new HashMap<>();
  config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
  AdminClient client = AdminClient.create(config);

  List<NewTopic> topics = new ArrayList<>();
  topics.add(new NewTopic(
      envProps.getProperty("input.topic.name"),
      Integer.parseInt(envProps.getProperty("input.topic.partitions")),
      Short.parseShort(envProps.getProperty("input.topic.replication.factor"))));
  topics.add(new NewTopic(
      envProps.getProperty("output.topic.name"),
      Integer.parseInt(envProps.getProperty("output.topic.partitions")),
      Short.parseShort(envProps.getProperty("output.topic.replication.factor"))));

  client.createTopics(topics);
  client.close();
}
 
Example #22
Source File: IntegrationTest.java    From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0 6 votes vote down vote up
@BeforeEach
void setUp() throws ExecutionException, InterruptedException {
    testBucketAccessor.clear(gcsPrefix);

    final Properties adminClientConfig = new Properties();
    adminClientConfig.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
    adminClient = AdminClient.create(adminClientConfig);

    final Map<String, Object> producerProps = new HashMap<>();
    producerProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafka.getBootstrapServers());
    producerProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.ByteArraySerializer");
    producerProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
            "org.apache.kafka.common.serialization.ByteArraySerializer");
    producer = new KafkaProducer<>(producerProps);

    final NewTopic newTopic0 = new NewTopic(TEST_TOPIC_0, 4, (short) 1);
    final NewTopic newTopic1 = new NewTopic(TEST_TOPIC_1, 4, (short) 1);
    adminClient.createTopics(Arrays.asList(newTopic0, newTopic1)).all().get();

    connectRunner = new ConnectRunner(pluginDir, kafka.getBootstrapServers(), OFFSET_FLUSH_INTERVAL_MS);
    connectRunner.start();
}
 
Example #23
Source File: SamplingUtils.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Build a wrapper around the topic with the given desired properties and {@link #DEFAULT_CLEANUP_POLICY}.
 *
 * @param topic The name of the topic.
 * @param partitionCount Desired partition count.
 * @param replicationFactor Desired replication factor.
 * @param retentionMs Desired retention in milliseconds.
 * @return A wrapper around the topic with the given desired properties.
 */
public static NewTopic wrapTopic(String topic, int partitionCount, short replicationFactor, long retentionMs) {
  if (partitionCount <= 0 || replicationFactor <= 0 || retentionMs <= 0) {
    throw new IllegalArgumentException(String.format("Partition count (%d), replication factor (%d), and retention ms (%d)"
                                                     + " must be positive for the topic (%s).", partitionCount,
                                                     replicationFactor, retentionMs, topic));
  }

  NewTopic newTopic = new NewTopic(topic, partitionCount, replicationFactor);
  Map<String, String> config = new HashMap<>(2);
  config.put(RetentionMsProp(), Long.toString(retentionMs));
  config.put(CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
  newTopic.configs(config);

  return newTopic;
}
 
Example #24
Source File: KafkaEmbedded.java    From micronaut-kafka with Apache License 2.0 6 votes vote down vote up
private void createTopics(int targetPort, Integer numPartitions) throws InterruptedException, java.util.concurrent.ExecutionException {
    List<String> topics = embeddedConfiguration.getTopics();

    if (LOG.isDebugEnabled()) {
        LOG.debug("Creating Kafka Topics in Embedded Kafka: {}", topics);
    }
    if (!topics.isEmpty()) {
        Properties properties = new Properties();
        properties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, ("127.0.0.1:" + targetPort));
        AdminClient adminClient = AdminClient.create(properties);
        final CreateTopicsResult result = adminClient.createTopics(topics.stream().map(s ->
                new NewTopic(s, numPartitions, (short) 1)).collect(Collectors.toList())
        );
        result.all().get();

        if (LOG.isInfoEnabled()) {
            LOG.info("Created Kafka Topics in Embedded Kafka: {}", topics);
        }
    }
}
 
Example #25
Source File: TopicAdmin.java    From kafka-message-tool with MIT License 6 votes vote down vote up
public void createNewTopic(TopicToAdd topicToAdd) throws Exception {
    final String topicName = topicToAdd.getTopicName();
    Logger.trace(String.format("Creating topic '%s' [partitions:%d, replication factor:%d, cleanup policy:%s]",
                               topicName,
                               topicToAdd.getPartitions(),
                               topicToAdd.getReplicationFactor(),
                               topicToAdd.getCleanupPolicy()));

    final NewTopic newTopic = new NewTopic(topicName,
                                           topicToAdd.getPartitions(),
                                           (short) topicToAdd.getReplicationFactor());
    newTopic.configs(topicConfigsMapFromTopicToAdd(topicToAdd));

    final CreateTopicsResult result = kafkaClientsAdminClient.createTopics(Collections.singletonList(newTopic));
    interpretCreateTopicResult(topicName, result);
}
 
Example #26
Source File: KafkaRule.java    From kafka-pubsub-emulator with Apache License 2.0 6 votes vote down vote up
public void createTopic(String name) throws Exception {
  try (AdminClient admin = AdminClient.create(getAdminConfig())) {
    int partitions = 3;
    int replication = this.replicationFactor;
    Matcher matcher = Pattern.compile("(-\\d+[p|r])").matcher(name);
    while (matcher.find()) {
      String group = matcher.group();
      if (group.endsWith("p")) {
        partitions = getNumber(group);
      } else {
        replication = getNumber(group);
      }
    }

    NewTopic topic = new NewTopic(name, partitions, (short) replication);
    CreateTopicsResult topicsResult = admin.createTopics(singleton(topic));
    topicsResult.all().get(5, TimeUnit.SECONDS);
  } catch (Exception e) {
    LOGGER.log(Level.SEVERE, "Error on create topics " + name, e);
    throw e;
  }
}
 
Example #27
Source File: MockKafka.java    From strimzi-kafka-operator with Apache License 2.0 6 votes vote down vote up
@Override
public Future<Void> createTopic(Topic t) {
    NewTopic newTopic = TopicSerialization.toNewTopic(t, null);
    Future<Void> event = createTopicResponse.apply(newTopic.name());
    if (event.succeeded()) {
        Topic.Builder topicBuilder = new Topic.Builder()
                .withTopicName(newTopic.name())
                .withNumPartitions(newTopic.numPartitions())
                .withNumReplicas(newTopic.replicationFactor())
                .withMetadata(t.getMetadata());
        try {
            Field field = NewTopic.class.getDeclaredField("configs");
            field.setAccessible(true);
            topicBuilder.withConfig((Map) field.get(newTopic));
        } catch (ReflectiveOperationException e) {
            throw new RuntimeException(e);
        }
        Topic topic = topicBuilder.build();
        topics.put(topic.getTopicName(), topic);
    }
    return event;
}
 
Example #28
Source File: NewTopicProperties.java    From stream-registry with Apache License 2.0 6 votes vote down vote up
public NewTopic buildNewTopic(final String topicName) {
  // We execute this method and its validations only when 'notification.events.kafka.topic.setup' is true
  // during KafkaSetupHandler building. If we use @Validated and its constraints, bean loading will fail even
  // when topic setup is disabled.
  val component = "enabled Kafka topic setup";

  Objects.requireNonNull(topicName, getWarningMessageOnNotDefinedProp(component, KAFKA_TOPIC_NAME_PROPERTY));
  Objects.requireNonNull(numPartitions, getWarningMessageOnNotDefinedProp(component,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".numPartitions")));
  Objects.requireNonNull(replicationFactor, getWarningMessageOnNotDefinedProp(component,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".replicationFactor")));

  val gtZeroWarning = " must be greater than zero";

  Preconditions.checkArgument(numPartitions.compareTo(0) > 0,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".numPartitions").concat(gtZeroWarning));
  Preconditions.checkArgument(replicationFactor.intValue() > 0,
      KAFKA_TOPIC_SETUP_PROPERTY.concat(".replicationFactor").concat(gtZeroWarning));

  return new NewTopic(topicName, numPartitions, replicationFactor).configs(configs);
}
 
Example #29
Source File: KafkaTopicClientImplTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldSetTopicCleanupPolicyToCompact() throws InterruptedException,
                                                          ExecutionException {
  expect(adminClient.listTopics()).andReturn(getEmptyListTopicResult());

  // Verify that the new topic configuration being passed to the admin client is what we expect.
  NewTopic newTopic = new NewTopic(topicName1, 1, (short) 1);
  newTopic.configs(Collections.singletonMap("cleanup.policy", "compact"));
  expect(adminClient.createTopics(singleNewTopic(newTopic))).andReturn(getCreateTopicsResult());
  replay(adminClient);

  KafkaTopicClient kafkaTopicClient = new KafkaTopicClientImpl(adminClient);
  kafkaTopicClient.createTopic(topicName1,
                               1,
                               (short) 1,
                               Collections.singletonMap("cleanup.policy", "compact"));
  verify(adminClient);
}
 
Example #30
Source File: KafkaTopicClientImplTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
private static Collection<NewTopic> singleNewTopic(final NewTopic expected) {
  class NewTopicsMatcher implements IArgumentMatcher {
    @SuppressWarnings("unchecked")
    @Override
    public boolean matches(final Object argument) {
      final Collection<NewTopic> newTopics = (Collection<NewTopic>) argument;
      if (newTopics.size() != 1) {
        return false;
      }

      final NewTopic actual = newTopics.iterator().next();
      return Objects.equals(actual.name(), expected.name())
             && Objects.equals(actual.replicationFactor(), expected.replicationFactor())
             && Objects.equals(actual.numPartitions(), expected.numPartitions())
             && Objects.equals(actual.configs(), expected.configs());
    }

    @Override
    public void appendTo(final StringBuffer buffer) {
      buffer.append("{NewTopic").append(expected).append("}");
    }
  }

  EasyMock.reportMatcher(new NewTopicsMatcher());
  return null;
}