Java Code Examples for kafka.admin.AdminUtils#fetchEntityConfig()
The following examples show how to use
kafka.admin.AdminUtils#fetchEntityConfig() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaAdminClient.java From data-highway with Apache License 2.0 | 6 votes |
public KafkaTopicDetails topicDetails(String topic) { Map<Object, List<Object>> map = getPartitionInfo(topic); int numPartitions = map.size(); int numReplicas = map.get(0).size(); RoadType type; Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic); String cleanupPolicyConfig = topicConfig .getProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE); switch (cleanupPolicyConfig) { case TopicConfig.CLEANUP_POLICY_COMPACT: type = RoadType.COMPACT; break; default: type = RoadType.NORMAL; } log.debug("numPartitions: {}, numReplicas: {}", numPartitions, numReplicas); return new KafkaTopicDetails(type, numPartitions, numReplicas); }
Example 2
Source File: TrafficControlIntegrationTest.java From data-highway with Apache License 2.0 | 6 votes |
@Test public void topic_updated_with_throttle_props() throws Exception { kafka.createTopic("test_topic4", 4, 1); JsonNode model = mapper .readTree( "{\"topicName\":\"test_topic4\",\"status\":{\"topicCreated\":true,\"partitions\":4,\"replicationFactor\":1}}"); KafkaModelReader modelReader = context.getBean(KafkaModelReader.class); TrafficControl agent = context.getBean(TrafficControl.class); agent.inspectModel("test", modelReader.read(model)); ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false); Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "test_topic4"); zkUtils.close(); assertThat(config.getProperty("leader.replication.throttled.replicas"), is("*")); assertThat(config.getProperty("follower.replication.throttled.replicas"), is("*")); }
Example 3
Source File: TrafficControlIntegrationTest.java From data-highway with Apache License 2.0 | 6 votes |
@Test public void topic_created_with_throttle_props() throws Exception { JsonNode model = mapper .readTree( "{\"topicName\":\"test_topic5\",\"status\":{\"topicCreated\":true,\"partitions\":4,\"replicationFactor\":1}}"); KafkaModelReader modelReader = context.getBean(KafkaModelReader.class); TrafficControl agent = context.getBean(TrafficControl.class); agent.inspectModel("test", modelReader.read(model)); ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false); Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "test_topic5"); zkUtils.close(); assertThat(config.getProperty("leader.replication.throttled.replicas"), is("*")); assertThat(config.getProperty("follower.replication.throttled.replicas"), is("*")); }
Example 4
Source File: KafkaStoreUtils.java From data-highway with Apache License 2.0 | 6 votes |
@SuppressWarnings({ "rawtypes", "unchecked" }) private static void verifyTopic(ZkUtils zkUtils, String topic) { Set topics = new HashSet(); topics.add(topic); // check # partition and the replication factor scala.collection.mutable.Map partitionAssignmentForTopics = zkUtils .getPartitionAssignmentForTopics(JavaConversions.asScalaSet(topics).toSeq()); scala.collection.Map partitionAssignment = (scala.collection.Map) partitionAssignmentForTopics.get(topic).get(); if (partitionAssignment.size() != 1) { throw new RuntimeException(String.format("The schema topic %s should have only 1 partition.", topic)); } // check the retention policy Properties prop = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic); String retentionPolicy = prop.getProperty(LogConfig.CleanupPolicyProp()); if (retentionPolicy == null || "compact".compareTo(retentionPolicy) != 0) { throw new RuntimeException(String.format("The retention policy of the schema topic %s must be compact.", topic)); } }
Example 5
Source File: KafkaAdminClient.java From data-highway with Apache License 2.0 | 5 votes |
public void checkAndUpdateTopic(KafkaRoad road) throws KafkaException { Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), road.getTopicName()); boolean modified = false; modified |= checkAndUpdateThrottledReplicas(topicConfig, LEADER_THROTTLED_REPLICAS); modified |= checkAndUpdateThrottledReplicas(topicConfig, FOLLOWER_THROTTLED_REPLICAS); if (modified) { AdminUtils.changeTopicConfig(zkUtils, road.getTopicName(), topicConfig); log.info("Updated topic {}", road.getTopicName()); } }
Example 6
Source File: TrafficControlIntegrationTest.java From data-highway with Apache License 2.0 | 5 votes |
@Test public void compact_road_created_correctly() throws Exception { KafkaRoad model = new KafkaRoad("test_topic6", "road.test_topic6", RoadType.COMPACT, null, null, false); context.getBean(TrafficControl.class).newModel("test_topic6", model); ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false); Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "road.test_topic6"); zkUtils.close(); assertThat(config.getProperty(TopicConfig.CLEANUP_POLICY_CONFIG), is(TopicConfig.CLEANUP_POLICY_COMPACT)); }
Example 7
Source File: KafkaTransportProviderAdmin.java From brooklin with BSD 2-Clause "Simplified" License | 5 votes |
/** * Consult Kafka to get the retention for a topic. This is not cached * in case the retention is changed externally after creation. * If no topic-level retention is configured, this method returns null. * * @param datastream Datastream * @return topic retention or null if no such config */ @Override public Duration getRetention(Datastream datastream) { Validate.isTrue(_zkUtils.isPresent(), "zkUtils should be present"); String destination = datastream.getDestination().getConnectionString(); Validate.notNull(destination, "null destination URI"); String topicName = KafkaTransportProviderUtils.getTopicName(destination); Properties props = AdminUtils.fetchEntityConfig(_zkUtils.get(), ConfigType.Topic(), topicName); if (!props.containsKey(TOPIC_RETENTION_MS)) { return null; } return Duration.ofMillis(Long.parseLong(props.getProperty(TOPIC_RETENTION_MS))); }
Example 8
Source File: KafkaBrokerMonitor.java From data-highway with Apache License 2.0 | 4 votes |
Properties getConfig(String brokerId) { return AdminUtils.fetchEntityConfig(zkUtils, BROKER_TYPE, brokerId); }