Java Code Examples for kafka.admin.AdminUtils

The following examples show how to use kafka.admin.AdminUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: data-highway   Source File: KafkaAdminClient.java    License: Apache License 2.0 6 votes vote down vote up
public void createTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = new Properties(defaultTopicConfig);
  topicConfig.setProperty(LEADER_THROTTLED_REPLICAS, WILDCARD);
  topicConfig.setProperty(FOLLOWER_THROTTLED_REPLICAS, WILDCARD);
  RoadType roadType = ofNullable(road.getType()).orElse(RoadType.NORMAL);
  switch (roadType) {
  case NORMAL:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
    break;
  case COMPACT:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);
    break;
  default:
    throw new KafkaException("Unhandled road type \"" + road.getType().name() + "\"");
  }
  AdminUtils
      .createTopic(zkUtils, road.getTopicName(), partitions, replicationFactor, topicConfig, DEFAULT_RACK_AWARE_MODE);
  log.info("Created {} topic {}", roadType, road.getTopicName());
}
 
Example 2
Source Project: data-highway   Source File: KafkaAdminClient.java    License: Apache License 2.0 6 votes vote down vote up
public KafkaTopicDetails topicDetails(String topic) {
  Map<Object, List<Object>> map = getPartitionInfo(topic);
  int numPartitions = map.size();
  int numReplicas = map.get(0).size();

  RoadType type;
  Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), topic);
  String cleanupPolicyConfig = topicConfig
      .getProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
  switch (cleanupPolicyConfig) {
  case TopicConfig.CLEANUP_POLICY_COMPACT:
    type = RoadType.COMPACT;
    break;
  default:
    type = RoadType.NORMAL;
  }
  log.debug("numPartitions: {}, numReplicas: {}", numPartitions, numReplicas);
  return new KafkaTopicDetails(type, numPartitions, numReplicas);
}
 
Example 3
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 4
@Test
public void topic_updated_with_throttle_props() throws Exception {
  kafka.createTopic("test_topic4", 4, 1);
  JsonNode model = mapper
      .readTree(
          "{\"topicName\":\"test_topic4\",\"status\":{\"topicCreated\":true,\"partitions\":4,\"replicationFactor\":1}}");
  KafkaModelReader modelReader = context.getBean(KafkaModelReader.class);
  TrafficControl agent = context.getBean(TrafficControl.class);
  agent.inspectModel("test", modelReader.read(model));

  ZkUtils zkUtils = ZkUtils.apply(kafka.zKConnectString(), 60000, 60000, false);
  Properties config = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), "test_topic4");
  zkUtils.close();

  assertThat(config.getProperty("leader.replication.throttled.replicas"), is("*"));
  assertThat(config.getProperty("follower.replication.throttled.replicas"), is("*"));
}
 
Example 5
public void createTopic(String topic) {
    ZkClient zkClient = new ZkClient(
            kafkaRpcConfig.getZookeeperConnect(),
            kafkaRpcConfig.getSessionTimeout(),
            kafkaRpcConfig.getConnectionTimeout(),
            ZKStringSerializer$.MODULE$);
    try {
        ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(kafkaRpcConfig.getZookeeperConnect()), false);
        Properties topicConfig = kafkaRpcConfig.topicProps();
        if (!AdminUtils.topicExists(zkUtils, topic)) {
            AdminUtils.createTopic(zkUtils, topic, kafkaRpcConfig.getNumPartitions(), 
                    kafkaRpcConfig.getReplicationFactor(), topicConfig, RackAwareMode.Enforced$.MODULE$);
        }
    } finally {
        zkClient.close();
    }
}
 
Example 6
Source Project: data-highway   Source File: KafkaStoreUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static void checkAndCreateTopic(String zkConnect, String topic, int replicas) {
  ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkConnect), IS_SECURE_KAFKA_CLUSTER);

  if (AdminUtils.topicExists(zkUtils, topic)) {
    verifyTopic(zkUtils, topic);
    return;
  }

  int partitions = 1;
  Properties topicConfig = new Properties();
  topicConfig.put(LogConfig.CleanupPolicyProp(), "compact");

  AdminUtils.createTopic(zkUtils, topic, partitions, replicas, topicConfig, RackAwareMode.Enforced$.MODULE$);

  zkClient.close();
  zkUtils.close();
}
 
Example 7
Source Project: Flink-CEPplus   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata();
			firstPart = partitionMetadata.head();
		}
		while (firstPart.errorCode() != 0);

		return firstPart.leader().get().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 8
Source Project: Flink-CEPplus   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkClient zkClient = createZkClient();
	PartitionMetadata firstPart = null;
	do {
		if (firstPart != null) {
			LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
			// not the first try. Sleep a bit
			Thread.sleep(150);
		}

		Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
		firstPart = partitionMetadata.head();
	}
	while (firstPart.errorCode() != 0);
	zkClient.close();

	return firstPart.leader().get().id();
}
 
Example 10
Source Project: Flink-CEPplus   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 11
Source Project: incubator-samoa   Source File: KafkaUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example 12
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 13
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkClient zkClient = createZkClient();
	PartitionMetadata firstPart = null;
	do {
		if (firstPart != null) {
			LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
			// not the first try. Sleep a bit
			Thread.sleep(150);
		}

		Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
		firstPart = partitionMetadata.head();
	}
	while (firstPart.errorCode() != 0);
	zkClient.close();

	return firstPart.leader().get().id();
}
 
Example 14
Source Project: oryx   Source File: KafkaUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...]
 * @param topic topic to create (if not already existing)
 * @param partitions number of topic partitions
 * @param topicProperties optional topic config properties
 */
public static void maybeCreateTopic(String zkServers,
                                    String topic,
                                    int partitions,
                                    Properties topicProperties) {
  ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false);
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      log.info("No need to create topic {} as it already exists", topic);
    } else {
      log.info("Creating topic {} with {} partition(s)", topic, partitions);
      try {
        AdminUtils.createTopic(
            zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$);
        log.info("Created topic {}", topic);
      } catch (TopicExistsException re) {
        log.info("Topic {} already exists", topic);
      }
    }
  } finally {
    zkUtils.close();
  }
}
 
Example 15
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		MetadataResponse.PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.error().code());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
			firstPart = partitionMetadata.get(0);
		}
		while (firstPart.error().code() != 0);

		return firstPart.leader().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 16
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		MetadataResponse.PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.error().code());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
			firstPart = partitionMetadata.get(0);
		}
		while (firstPart.error().code() != 0);

		return firstPart.leader().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 17
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 18
Source Project: rya   Source File: KafkaTopics.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a set of Kafka topics for each topic that does not already exist.
 *
 * @param zookeeperServers - The Zookeeper servers that are used by the Kafka Streams program. (not null)
 * @param topicNames - The topics that will be created. (not null)
 * @param partitions - The number of partitions that each of the topics will have.
 * @param replicationFactor - The replication factor of the topics that are created.
 * @param topicProperties - The optional properties of the topics to create.
 */
public static void createTopics(
        final String zookeeperServers,
        final Set<String> topicNames,
        final int partitions,
        final int replicationFactor,
        final Optional<Properties> topicProperties) {
    requireNonNull(zookeeperServers);
    requireNonNull(topicNames);

    ZkUtils zkUtils = null;
    try {
        zkUtils = ZkUtils.apply(new ZkClient(zookeeperServers, 30000, 30000, ZKStringSerializer$.MODULE$), false);
        for(final String topicName : topicNames) {
            if(!AdminUtils.topicExists(zkUtils, topicName)) {
                AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, topicProperties.orElse(new Properties()), RackAwareMode.Disabled$.MODULE$);
            }
        }
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 
Example 19
Source Project: dk-fitting   Source File: Topicutil.java    License: Apache License 2.0 6 votes vote down vote up
/**
     * 查询所有topic,包括已经被标记删除,还没有删除的topic。
     * @return topic的list
     */
    public static List<String> queryAllTopic(){
        ZkUtils zkUtils = ZkUtils.apply(zkUrl, sessionTimeout, connectionTimeout, JaasUtils.isZkSecurityEnabled());
        ArrayList<String> topics = new ArrayList<String>();
//        AdminUtils.topicExists()
        scala.collection.Map<String, Properties> stringPropertiesMap = AdminUtils.fetchAllTopicConfigs(zkUtils);
        Map<String, Properties> javaMap = JavaConversions.mapAsJavaMap(stringPropertiesMap);
        Iterator<String> iterator = javaMap.keySet().iterator();
        while(iterator.hasNext()){
            String key = iterator.next();
            Properties properties = javaMap.get(key);
            topics.add(key);
        }
        zkUtils.close();
        return  topics;
    }
 
Example 20
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic,
                        int partitions,
                        int replication,
                        Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);
  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // registerTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), false);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 
Example 21
Source Project: hermes   Source File: TopicTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void createTopicInTestEnv() {
	String ZOOKEEPER_CONNECT = "";
	ZkClient zkClient = new ZkClient(new ZkConnection(ZOOKEEPER_CONNECT));
	ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(ZOOKEEPER_CONNECT), false);
	zkClient.setZkSerializer(new ZKStringSerializer());
	int partition = 1;
	int replication = 1;
	String topic = String.format("kafka.test_create_topic_p%s_r%s", partition, replication);
	if (AdminUtils.topicExists(zkUtils, topic)) {
		TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils);
		System.out.println(topicMetadata);
		AdminUtils.deleteTopic(zkUtils, topic);
	}
	AdminUtils.createTopic(zkUtils, topic, partition, replication, new Properties());
}
 
Example 22
Source Project: datacollector   Source File: KafkaProducer09IT.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws Exception {
  int zkConnectionTimeout = 6000;
  int zkSessionTimeout = 6000;

  zookeeper = new EmbeddedZookeeper();
  zkConnect = String.format("127.0.0.1:%d", zookeeper.port());
  zkUtils = ZkUtils.apply(
      zkConnect, zkSessionTimeout, zkConnectionTimeout,
      JaasUtils.isZkSecurityEnabled());

  port = NetworkUtils.getRandomPort();
  kafkaServer = TestUtil09.createKafkaServer(port, zkConnect);
  for (int i = 0; i < topics.length; i++) {
    topics[i] = UUID.randomUUID().toString();
    AdminUtils.createTopic(zkUtils, topics[i], 1, 1, new Properties());

    TestUtils.waitUntilMetadataIsPropagated(
        scala.collection.JavaConversions.asScalaBuffer(Arrays.asList(kafkaServer)),
        topics[i], 0, 5000);
  }
}
 
Example 23
private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}
 
Example 24
Source Project: pubsub   Source File: KafkaResourceController.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void startAction() throws Exception {
  ZkClient zookeeperClient =
      new ZkClient(
          KafkaFlags.getInstance().zookeeperIp, 15000, 10000, ZKStringSerializer$.MODULE$);
  ZkUtils zookeeperUtils =
      new ZkUtils(zookeeperClient, new ZkConnection(KafkaFlags.getInstance().zookeeperIp), false);
  try {
    deleteTopic(zookeeperUtils);
    AdminUtils.createTopic(
        zookeeperUtils,
        topic,
        KafkaFlags.getInstance().partitions,
        KafkaFlags.getInstance().replicationFactor,
        AdminUtils.createTopic$default$5(),
        AdminUtils.createTopic$default$6());
    log.info("Created topic " + topic + ".");
  } finally {
    zookeeperClient.close();
  }
}
 
Example 25
Source Project: flink   Source File: KafkaTestEnvironmentImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 26
/**
 * Create Kafka topic based on the destination connection string, if it does not already exist.
 * @param connectionString connection string from which to obtain topic name
 * @param numberOfPartitions number of partitions
 * @param topicConfig topic config to use for topic creation
 */
public void createTopic(String connectionString, int numberOfPartitions, Properties topicConfig) {
  Validate.notNull(connectionString, "destination should not be null");
  Validate.notNull(topicConfig, "topicConfig should not be null");
  Validate.isTrue(_zkUtils.isPresent(), "zkUtils should be present");

  String topicName = KafkaTransportProviderUtils.getTopicName(connectionString);
  populateTopicConfig(topicConfig);
  try {
    // Create only if it doesn't exist.
    if (!AdminUtils.topicExists(_zkUtils.get(), topicName)) {
      int replicationFactor = Integer.parseInt(topicConfig.getProperty("replicationFactor", DEFAULT_REPLICATION_FACTOR));
      LOG.info("Creating topic with name {} partitions={} with properties {}", topicName, numberOfPartitions,
              topicConfig);

      AdminUtils.createTopic(_zkUtils.get(), topicName, numberOfPartitions, replicationFactor, topicConfig, RackAwareMode.Disabled$.MODULE$);
    } else {
      LOG.warn("Topic with name {} already exists", topicName);
    }
  } catch (Throwable e) {
    LOG.error("Creating topic {} failed with exception {}", topicName, e);
    throw e;
  }
}
 
Example 27
Source Project: ameliant-tools   Source File: EmbeddedKafkaBroker.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void before() throws Throwable {
    logDirectory = tempDir(perTest("kafka-log"));
    Properties properties = brokerDefinition.getProperties();
    properties.setProperty(KafkaConfig.LogDirProp(), logDirectory.getCanonicalPath());
    kafkaServer = new KafkaServer(new KafkaConfig(properties),
            SystemTime$.MODULE$, Some$.MODULE$.apply("kafkaServer"));
    kafkaServer.startup();

    List<TopicDefinition> topicDefinitions = brokerDefinition.getTopicDefinitions();
    if (!topicDefinitions.isEmpty()) {
        ZkUtils zkUtils = ZkUtils.apply(brokerDefinition.getZookeeperConnect(), 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        for (TopicDefinition topicDefinition : topicDefinitions) {
            String name = topicDefinition.getName();
            log.info("Creating topic {}", name);
            AdminUtils.createTopic(zkUtils,
                    name,
                    topicDefinition.getPartitions(),
                    topicDefinition.getReplicationFactor(),
                    topicDefinition.getProperties());
        }
    }
}
 
Example 28
Source Project: incubator-gobblin   Source File: KafkaTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public void provisionTopic(String topic) {
  if (_topicConsumerMap.containsKey(topic)) {
    // nothing to do: return
  } else {
    // provision topic
    AdminUtils.createTopic(ZkUtils.apply(_kafkaServerSuite.getZkClient(), false),
        topic, 1, 1, new Properties());

    List<KafkaServer> servers = new ArrayList<>();
    servers.add(_kafkaServerSuite.getKafkaServer());
    kafka.utils.TestUtils.waitUntilMetadataIsPropagated(scala.collection.JavaConversions.asScalaBuffer(servers), topic, 0, 5000);
    KafkaConsumerSuite consumerSuite = new KafkaConsumerSuite(_kafkaServerSuite.getZkConnectString(), topic);
    _topicConsumerMap.put(topic, consumerSuite);
  }
}
 
Example 29
Source Project: data-highway   Source File: KafkaAdminClient.java    License: Apache License 2.0 5 votes vote down vote up
public void checkAndUpdateTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), road.getTopicName());
  boolean modified = false;
  modified |= checkAndUpdateThrottledReplicas(topicConfig, LEADER_THROTTLED_REPLICAS);
  modified |= checkAndUpdateThrottledReplicas(topicConfig, FOLLOWER_THROTTLED_REPLICAS);
  if (modified) {
    AdminUtils.changeTopicConfig(zkUtils, road.getTopicName(), topicConfig);
    log.info("Updated topic {}", road.getTopicName());
  }
}
 
Example 30
Source Project: kylin   Source File: MockKafka.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Delete may not work
 *
 * @param topic
 */
public void deleteTopic(String topic) {
    ZkClient zkClient = new ZkClient(zkConnection);
    ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
    zkClient.setZkSerializer(new ZKStringSerializer());
    AdminUtils.deleteTopic(zkUtils, topic);
    zkClient.close();
}