Java Code Examples for org.apache.kafka.clients.admin.AdminClient#create()

private KafkaFactory.KafkaClientModel get(){
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "10.104.156.83:9092");
    props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    props.put("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    props.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, "SASL_PLAINTEXT");
    props.put(SaslConfigs.SASL_MECHANISM, "PLAIN");
    props.put("acks", "all");
    props.put("retries", 0);
    props.put("batch.size", 16384);
    props.put("linger.ms", 1);
    props.put("buffer.memory", 33554432);
    props.put("sasl.jaas.config",
            "org.apache.kafka.common.security.plain.PlainLoginModule required username='kafka' password='kafka';");
    KafkaProducer<String, Byte[]> producer = new KafkaProducer<>(props);
    AdminClient client = AdminClient.create(props);

    KafkaFactory.KafkaClientModel kafkaClientModel = new KafkaFactory.KafkaClientModel(producer, client);
    return kafkaClientModel;
}
 

private void createTopics(String bootstrapServers) {
    Properties properties = new Properties();
    properties.put("bootstrap.servers", bootstrapServers);
    properties.put("connections.max.idle.ms", 10000);
    properties.put("request.timeout.ms", 5000);
    try (AdminClient client = AdminClient.create(properties)) {
        CreateTopicsResult result = client.createTopics(Arrays.asList(
                new NewTopic("storage-topic", 1, (short) 1),
                new NewTopic("global-id-topic", 1, (short) 1),
                new NewTopic("snapshot-topic", 1, (short) 1)
        ));
        try {
            result.all().get();
        } catch ( InterruptedException | ExecutionException e ) {
            throw new IllegalStateException(e);
        }
    }
}
 

@Override
public void initialize(File configurationFile, StatsLogger statsLogger) throws IOException {
    config = mapper.readValue(configurationFile, Config.class);

    Properties commonProperties = new Properties();
    commonProperties.load(new StringReader(config.commonConfig));

    producerProperties = new Properties();
    commonProperties.forEach((key, value) -> producerProperties.put(key, value));
    producerProperties.load(new StringReader(config.producerConfig));
    producerProperties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    producerProperties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());

    consumerProperties = new Properties();
    commonProperties.forEach((key, value) -> consumerProperties.put(key, value));
    consumerProperties.load(new StringReader(config.consumerConfig));
    consumerProperties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    consumerProperties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());

    topicProperties = new Properties();
    topicProperties.load(new StringReader(config.topicConfig));

    admin = AdminClient.create(commonProperties);

    producer = new KafkaProducer<>(producerProperties);
}
 

public void createTopic(String name) throws Exception {
  try (AdminClient admin = AdminClient.create(getAdminConfig())) {
    int partitions = 3;
    int replication = this.replicationFactor;
    Matcher matcher = Pattern.compile("(-\\d+[p|r])").matcher(name);
    while (matcher.find()) {
      String group = matcher.group();
      if (group.endsWith("p")) {
        partitions = getNumber(group);
      } else {
        replication = getNumber(group);
      }
    }

    NewTopic topic = new NewTopic(name, partitions, (short) replication);
    CreateTopicsResult topicsResult = admin.createTopics(singleton(topic));
    topicsResult.all().get(5, TimeUnit.SECONDS);
  } catch (Exception e) {
    LOGGER.log(Level.SEVERE, "Error on create topics " + name, e);
    throw e;
  }
}
 

public DefaultCollector()
        throws ConfigFileNotFoundException
{
    CollectorConfig config = CollectorConfig.INSTANCE();
    config.init();
    // init meta client
    metaClient = new MetaClient(config.getMetaServerHost(),
            config.getMetaServerPort());
    // init kafka admin client
    Properties properties = new Properties();
    properties.setProperty("bootstrap.servers", config.getKafkaBootstrapServers());
    properties.setProperty("client.id", "producerAdmin");
    properties.setProperty("metadata.max.age.ms", "3000");
    kafkaAdminClient = AdminClient.create(properties);
    this.collectorRuntime = new CollectorRuntime(config);
    init();
}
 

@Provides
@Singleton
AdminClient provideAdminClient(@BootstrapServers String bootstrapServers, @Nullable SslConfiguration sslConfiguration) {
    Properties properties = new Properties();
    properties.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);

    if (null != sslConfiguration) {
        properties.put(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, SslConfiguration.PROTOCOL);

        properties.put(SslConfigs.SSL_TRUSTSTORE_LOCATION_CONFIG, sslConfiguration.getTrustStoreLocation());
        properties.put(SslConfigs.SSL_TRUSTSTORE_PASSWORD_CONFIG, sslConfiguration.getTrustStorePassword());

        properties.put(SslConfigs.SSL_KEYSTORE_LOCATION_CONFIG, sslConfiguration.getKeyStoreLocation());
        properties.put(SslConfigs.SSL_KEYSTORE_PASSWORD_CONFIG, sslConfiguration.getKeyStorePassword());
        properties.put(SslConfigs.SSL_KEY_PASSWORD_CONFIG, sslConfiguration.getKeyPassword());
    }

    return AdminClient.create(properties);
}
 

/**
 * Get the data for the topic partition in the specified consumer group
 */
public Map<Integer, Long> getKafkaOffset(String clusterAlias, String group, String topic, Set<Integer> partitionids) {
	Map<Integer, Long> partitionOffset = new HashMap<>();
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));

	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(prop, clusterAlias);
	}
	AdminClient adminClient = null;
	try {
		adminClient = AdminClient.create(prop);
		List<TopicPartition> tps = new ArrayList<>();
		for (int partitionid : partitionids) {
			TopicPartition tp = new TopicPartition(topic, partitionid);
			tps.add(tp);
		}

		ListConsumerGroupOffsetsOptions consumerOffsetOptions = new ListConsumerGroupOffsetsOptions();
		consumerOffsetOptions.topicPartitions(tps);

		ListConsumerGroupOffsetsResult offsets = adminClient.listConsumerGroupOffsets(group, consumerOffsetOptions);
		for (Entry<TopicPartition, OffsetAndMetadata> entry : offsets.partitionsToOffsetAndMetadata().get().entrySet()) {
			if (topic.equals(entry.getKey().topic())) {
				partitionOffset.put(entry.getKey().partition(), entry.getValue().offset());
			}
		}
	} catch (Exception e) {
		LOG.error("Get consumer offset has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		adminClient.close();
	}
	return partitionOffset;
}
 

@Before
public void before() throws IOException {
  kafkaAdminClient = AdminClient.create(config());
  TopologyBuilderAdminClient adminClient = new TopologyBuilderAdminClient(kafkaAdminClient);
  adminClient.clearAcls();

  cs = new ClusterState();
  aclsProvider = new SimpleAclsProvider(adminClient);
  accessControlManager = new AccessControlManager(aclsProvider, cs);
}
 

@Before
public void setUp() {
  testTopic = UUID.randomUUID().toString();
  KAFKA.createTopic(testTopic);

  adminClient = AdminClient.create(ImmutableMap.of(
      AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA.bootstrapServers()));

  client = new KafkaTopicClientImpl(adminClient);
}
 

/** Alter topic config. */
public String changeTopicConfig(String clusterAlias, String topic, String type, ConfigEntry configEntry) {
	JSONObject object = new JSONObject();
	Properties prop = new Properties();
	prop.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, parseBrokerServer(clusterAlias));
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		kafkaService.sasl(prop, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		kafkaService.ssl(prop, clusterAlias);
	}
	try {
		switch (type) {
		case Topic.ADD:
			AdminClient adminClientAdd = AdminClient.create(prop);
			object.put("type", type);
			object.put("value", addTopicConfig(clusterAlias, adminClientAdd, topic, configEntry));
			adminClientAdd.close();
			break;
		case Topic.DELETE:
			AdminClient adminClientDelete = AdminClient.create(prop);
			object.put("type", type);
			object.put("value", deleteTopicConfig(clusterAlias, adminClientDelete, topic, configEntry));
			adminClientDelete.close();
			break;
		case Topic.DESCRIBE:
			object.put("type", type);
			object.put("value", describeTopicConfig(clusterAlias, topic));
			break;
		default:
			break;
		}

	} catch (Exception e) {
		e.printStackTrace();
		LOG.error("Type[" + type + "] topic config has error, msg is " + e.getMessage());
	}
	return object.toJSONString();
}
 

private AdminClient createAdminClient() {
  val configs = new HashMap<String, Object>();
  configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);

  log.info("Creating a Kafka Admin client with configuration {}", configs);

  return AdminClient.create(configs);
}
 

private AdminClient createAdminClient() {
  Map<String, Object> configs = new HashMap<>();
  configs.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);

  log.info("Creating a Kafka Admin client with configuration {}", configs);

  return AdminClient.create(configs);
}
 

public static KsqlContext create(
    KsqlConfig ksqlConfig,
    SchemaRegistryClient schemaRegistryClient
) {
  if (ksqlConfig == null) {
    ksqlConfig = new KsqlConfig(Collections.emptyMap());
  }
  Map<String, Object> streamsProperties = ksqlConfig.getKsqlStreamConfigProps();
  if (!streamsProperties.containsKey(StreamsConfig.APPLICATION_ID_CONFIG)) {
    streamsProperties.put(StreamsConfig.APPLICATION_ID_CONFIG, APPLICATION_ID_OPTION_DEFAULT);
  }
  if (!streamsProperties.containsKey(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG)) {
    streamsProperties.put(
        StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
        KAFKA_BOOTSTRAP_SERVER_OPTION_DEFAULT
    );
  }
  AdminClient adminClient = AdminClient.create(ksqlConfig.getKsqlAdminClientConfigProps());
  KafkaTopicClient topicClient = new KafkaTopicClientImpl(adminClient);
  if (schemaRegistryClient == null) {
    return new KsqlContext(adminClient, topicClient, new KsqlEngine(ksqlConfig, topicClient));
  } else {
    return new KsqlContext(
        adminClient,
        topicClient,
        new KsqlEngine(
            ksqlConfig,
            topicClient,
            schemaRegistryClient,
            new MetaStoreImpl()
        )
    );
  }

}
 

@Bean
public AdminClient kafkaAdminClient(KafkaContainer kafkaContainer) {
    Map<String, Object> adminProperties = new HashMap<>();
    adminProperties.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaContainer.getBootstrapServers());

    return AdminClient.create(adminProperties);
}
 

public static void runRecipe(final String configPath) throws IOException {

    Properties envProps = AggregatingMinMax.loadPropertiesFromConfigFile(configPath);

    try ( AdminClient client = AdminClient.create(
            Collections.singletonMap("bootstrap.servers", envProps.getProperty("bootstrap.servers")))) {
      createKafkaTopicsInCluster(client, envProps);
    }

    Topology topology = AggregatingMinMax.buildTopology(
            new StreamsBuilder(),
            envProps,
            AggregatingMinMax.ticketSaleSerde(envProps),
            AggregatingMinMax.movieFiguresSerde(envProps));

    final KafkaStreams streams = new KafkaStreams(
            topology,
            AggregatingMinMax.buildStreamsProperties(envProps));
    final CountDownLatch latch = new CountDownLatch(1);

    // Attach shutdown handler to catch Control-C.
    Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") {
      @Override
      public void run() {
				streams.close(Duration.ofSeconds(5));
        latch.countDown();
      }
    });

    try {
      streams.start();
      latch.await();
    } catch (Throwable e) {
      System.exit(1);
    }
    System.exit(0);

  }
 

public void createTopics(Properties envProps) {
    Map<String, Object> config = new HashMap<>();
    config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));
    AdminClient client = AdminClient.create(config);

    List<NewTopic> topics = new ArrayList<>();

    topics.add(new NewTopic(
            envProps.getProperty("movie.topic.name"),
            Integer.parseInt(envProps.getProperty("movie.topic.partitions")),
            Short.parseShort(envProps.getProperty("movie.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("rekeyed.movie.topic.name"),
            Integer.parseInt(envProps.getProperty("rekeyed.movie.topic.partitions")),
            Short.parseShort(envProps.getProperty("rekeyed.movie.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("rating.topic.name"),
            Integer.parseInt(envProps.getProperty("rating.topic.partitions")),
            Short.parseShort(envProps.getProperty("rating.topic.replication.factor"))));

    topics.add(new NewTopic(
            envProps.getProperty("rated.movies.topic.name"),
            Integer.parseInt(envProps.getProperty("rated.movies.topic.partitions")),
            Short.parseShort(envProps.getProperty("rated.movies.topic.replication.factor"))));

    client.createTopics(topics);
    client.close();
}
 

public void deleteTopics(Properties envProps) {

        Map<String, Object> config = new HashMap<>();
        config.put("bootstrap.servers", envProps.getProperty("bootstrap.servers"));

        try (AdminClient adminClient = AdminClient.create(config)) {

            List<String> topics = new ArrayList<String>();
            topics.add(envProps.getProperty("input.topic.name"));
            topics.add(envProps.getProperty("output.topic.name"));
    
            adminClient.deleteTopics(topics);

        }

    }
 

public AdminClient getAdminClient(final Map<String, Object> config) {
  // create a new client upon each call; but expect this call to be only triggered once so this should be fine
  return AdminClient.create(config);
}
 

@Bean("adminClient")
public AdminClient adminClient(@Autowired @Qualifier("kafkaAdminProperties") Properties kafkaAdminProperties) {
    return AdminClient.create(kafkaAdminProperties);
}
 

@Test
public void shouldAggregateHoppingWindow() throws Exception {

  testHarness.publishTestData(topicName, dataProvider, now);


  final String streamName = "HOPPING_AGGTEST";

  final String queryString = String.format(
          "CREATE TABLE %s AS SELECT %s FROM ORDERS WINDOW %s WHERE ITEMID = 'ITEM_1' GROUP BY ITEMID;",
          streamName,
          "ITEMID, COUNT(ITEMID), SUM(ORDERUNITS), SUM(ORDERUNITS * 10)",
          "HOPPING ( SIZE 10 SECONDS, ADVANCE BY 5 SECONDS)"
  );

  ksqlContext.sql(queryString);

  Schema resultSchema = ksqlContext.getMetaStore().getSource(streamName).getSchema();


  final GenericRow expected = new GenericRow(Arrays.asList(null, null, "ITEM_1", 2 /** 2 x
   items **/, 20.0, 200.0));

  final Map<String, GenericRow> results = new HashMap<>();
  TestUtils.waitForCondition(() -> {
    final Map<Windowed<String>, GenericRow> windowedResults = testHarness.consumeData(streamName, resultSchema, 1, new TimeWindowedDeserializer<>(new StringDeserializer()), 1000);
    updateResults(results, windowedResults);
    final GenericRow actual = results.get("ITEM_1");
    return expected.equals(actual);
  }, 60000, "didn't receive correct results within timeout");

  AdminClient adminClient = AdminClient.create(testHarness.ksqlConfig.getKsqlStreamConfigProps());
  KafkaTopicClient topicClient = new KafkaTopicClientImpl(adminClient);

  Set<String> topicBeforeCleanup = topicClient.listTopicNames();

  assertThat("Expected to have 5 topics instead have : " + topicBeforeCleanup.size(),
             topicBeforeCleanup.size(), equalTo(5));
  QueryMetadata queryMetadata = ksqlContext.getRunningQueries().iterator().next();

  queryMetadata.close();
  Set<String> topicsAfterCleanUp = topicClient.listTopicNames();

  assertThat("Expected to see 3 topics after clean up but seeing " + topicsAfterCleanUp.size
      (), topicsAfterCleanUp.size(), equalTo(3));
  assertThat(topicClient.getTopicCleanupPolicy(streamName), equalTo(
      KafkaTopicClient.TopicCleanupPolicy.DELETE));
}