Java Code Examples for kafka.admin.AdminUtils

The following are top voted examples for showing how to use kafka.admin.AdminUtils. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: incubator-samoa   File: KafkaUtilsTest.java   View source code 10 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example 2
Project: incubator-samoa   File: KafkaDestinationProcessorTest.java   View source code 9 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topic
    AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example 3
Project: datacollector   File: KafkaProducer09IT.java   View source code 9 votes vote down vote up
@BeforeClass
public static void setUpClass() throws Exception {
  int zkConnectionTimeout = 6000;
  int zkSessionTimeout = 6000;

  zookeeper = new EmbeddedZookeeper();
  zkConnect = String.format("127.0.0.1:%d", zookeeper.port());
  zkUtils = ZkUtils.apply(
      zkConnect, zkSessionTimeout, zkConnectionTimeout,
      JaasUtils.isZkSecurityEnabled());

  port = NetworkUtils.getRandomPort();
  kafkaServer = TestUtil09.createKafkaServer(port, zkConnect);
  for (int i = 0; i < topics.length; i++) {
    topics[i] = UUID.randomUUID().toString();
    AdminUtils.createTopic(zkUtils, topics[i], 1, 1, new Properties());

    TestUtils.waitUntilMetadataIsPropagated(
        scala.collection.JavaConversions.asScalaBuffer(Arrays.asList(kafkaServer)),
        topics[i], 0, 5000);
  }
}
 
Example 4
Project: incubator-samoa   File: KafkaEntranceProcessorTest.java   View source code 7 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics        
    AdminUtils.createTopic(zkUtils, TOPIC_OOS, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example 5
Project: management-sdk-for-kafka   File: ClusterTools.java   View source code 6 votes vote down vote up
/**
 * Get topic configuration
 *
 * @param connection connection
 * @param topicName  topic name
 * @return topic properties
 */
public Properties getTopicProperties(final ZkUtils connection, final String topicName) {
    try {
        return AdminUtils.fetchEntityConfig(connection, ConfigType.Topic(), topicName);
    } catch (IllegalArgumentException | KafkaException e) {
        throw new TopicOperationException(topicName, e.getMessage(), e, this.getClass());
    }
}
 
Example 6
Project: management-sdk-for-kafka   File: ClusterTools.java   View source code 6 votes vote down vote up
/**
 * Create a topic
 *
 * @param connection        Connection
 * @param topicName         Topic name
 * @param partitions        The number of partitions for the topic being created
 * @param replicationFactor The replication factor for each partition in the topic being created
 * @param topicProperties   A topic configuration override for an existing topic
 * @throws TopicOperationException if topic was not created.
 */
public void createTopic(final ZkUtils connection, final String topicName,
                        final int partitions,
                        final int replicationFactor,
                        final Properties topicProperties) {

    try {
        AdminUtils.createTopic(connection,
                topicName,
                partitions,
                replicationFactor,
                topicProperties);

    } catch (IllegalArgumentException | KafkaException | AdminOperationException e) {
        throw new TopicOperationException(topicName, e.getMessage(), e, this.getClass());
    }
}
 
Example 7
Project: cruise-control   File: KafkaSampleStore.java   View source code 6 votes vote down vote up
private void ensureTopicCreated(Map<String, ?> config) {
  ZkUtils zkUtils = createZkUtils(config);
  Map<String, List<PartitionInfo>> topics = _consumers.get(0).listTopics();
  long snapshotWindowMs = Long.parseLong((String) config.get(KafkaCruiseControlConfig.LOAD_SNAPSHOT_WINDOW_MS_CONFIG));
  int numSnapshotWindows = Integer.parseInt((String) config.get(KafkaCruiseControlConfig.NUM_LOAD_SNAPSHOTS_CONFIG));
  long retentionMs = (numSnapshotWindows * ADDITIONAL_SNAPSHOT_WINDOW_TO_RETAIN_FACTOR) * snapshotWindowMs;
  Properties props = new Properties();
  props.setProperty(LogConfig.RetentionMsProp(), Long.toString(retentionMs));
  props.setProperty(LogConfig.CleanupPolicyProp(), DEFAULT_CLEANUP_POLICY);
  int replicationFactor = Math.min(2, zkUtils.getAllBrokersInCluster().size());
  if (!topics.containsKey(_partitionMetricSampleStoreTopic)) {
    AdminUtils.createTopic(zkUtils, _partitionMetricSampleStoreTopic, 32, replicationFactor, props, RackAwareMode.Safe$.MODULE$);
  } else {
    AdminUtils.changeTopicConfig(zkUtils, _partitionMetricSampleStoreTopic, props);
  }

  if (!topics.containsKey(_brokerMetricSampleStoreTopic)) {
    AdminUtils.createTopic(zkUtils, _brokerMetricSampleStoreTopic, 32, replicationFactor, props, RackAwareMode.Safe$.MODULE$);
  } else {
    AdminUtils.changeTopicConfig(zkUtils, _brokerMetricSampleStoreTopic, props);
  }

  KafkaCruiseControlUtils.closeZkUtilsWithTimeout(zkUtils, 10000);
}
 
Example 8
Project: kafka-0.11.0.0-src-with-comment   File: KafkaEmbedded.java   View source code 6 votes vote down vote up
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(final String topic,
                        final int partitions,
                        final int replication,
                        final Properties topicConfig) {
    log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
        topic, partitions, replication, topicConfig);

    // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
    // createTopic() will only seem to work (it will return without error).  The topic will exist in
    // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
    // topic.
    final ZkClient zkClient = new ZkClient(
        zookeeperConnect(),
        DEFAULT_ZK_SESSION_TIMEOUT_MS,
        DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
        ZKStringSerializer$.MODULE$);
    final boolean isSecure = false;
    final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
    AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
    zkClient.close();
}
 
Example 9
Project: kafka-streams-machine-learning-examples   File: KafkaEmbedded.java   View source code 6 votes vote down vote up
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic,
                        int partitions,
                        int replication,
                        Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);
  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // createTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  boolean isSecure = false;
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 
Example 10
Project: Availability-Monitor-for-Kafka   File: KafkaUtils.java   View source code 6 votes vote down vote up
/**
 * Creates a Topic.
 *
 * @param topicName             Topic name.
 * @param partitions        Number of partitions for the topic.
 * @param replicationFactor Replication factor.
 * @param curatorFramework CuratorFramework.
 */
public static void createTopic(String topicName, int partitions, int replicationFactor, CuratorFramework curatorFramework) {
    if (partitions <= 0)
        throw new AdminOperationException("number of partitions must be larger than 0");

    if (replicationFactor <= 0)
        throw new AdminOperationException("replication factor must be larger than 0");

    if (!topicExists(topicName, curatorFramework)) {
        m_logger.info(String.format("Topic %s not found, creating...", topicName));
        ZkClient zkClient = fromCurator(curatorFramework);
        try {
            AdminUtils.createTopic(zkClient, topicName, partitions, replicationFactor, new Properties());
            m_logger.info("Topic created. name: {}, partitions: {}, replicationFactor: {}", topicName,
                    partitions, replicationFactor);
        } catch (TopicExistsException ignore) {
            m_logger.info("Topic exists. name: {}", topicName);
        } finally {
            if (zkClient != null) {
                zkClient.close();
            }
        }
    } else {
        m_logger.info(String.format("Topic %s found!", topicName));
    }
}
 
Example 11
Project: kafka-testkit   File: KafkaEmbedded.java   View source code 6 votes vote down vote up
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(final String topic,
                        final int partitions,
                        final int replication,
                        final Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);

  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // createTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  final ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  final boolean isSecure = false;
  final ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 
Example 12
Project: kafka-monitor   File: MultiClusterTopicManagementService.java   View source code 6 votes vote down vote up
private static void reassignPartitions(ZkUtils zkUtils, Collection<Broker> brokers, String topic, int partitionCount, int replicationFactor) {
  scala.collection.mutable.ArrayBuffer<BrokerMetadata> brokersMetadata = new scala.collection.mutable.ArrayBuffer<>(brokers.size());
  for (Broker broker : brokers) {
    brokersMetadata.$plus$eq(new BrokerMetadata(broker.id(), broker.rack()));
  }
  scala.collection.Map<Object, Seq<Object>> newAssignment =
      AdminUtils.assignReplicasToBrokers(brokersMetadata, partitionCount, replicationFactor, 0, 0);

  scala.collection.mutable.ArrayBuffer<String> topicList = new scala.collection.mutable.ArrayBuffer<>();
  topicList.$plus$eq(topic);
  scala.collection.Map<Object, scala.collection.Seq<Object>> currentAssignment = zkUtils.getPartitionAssignmentForTopics(topicList).apply(topic);
  String currentAssignmentJson = formatAsReassignmentJson(topic, currentAssignment);
  String newAssignmentJson = formatAsReassignmentJson(topic, newAssignment);

  LOG.info("Reassign partitions for topic " + topic);
  LOG.info("Current partition replica assignment " + currentAssignmentJson);
  LOG.info("New partition replica assignment " + newAssignmentJson);
  zkUtils.createPersistentPath(ZkUtils.ReassignPartitionsPath(), newAssignmentJson, zkUtils.DefaultAcls());
}
 
Example 13
Project: kafka-monitor   File: Utils.java   View source code 6 votes vote down vote up
/**
 * Create the topic that the monitor uses to monitor the cluster.  This method attempts to create a topic so that all
 * the brokers in the cluster will have partitionToBrokerRatio partitions.  If the topic exists, but has different parameters
 * then this does nothing to update the parameters.
 *
 * TODO: Do we care about rack aware mode?  I would think no because we want to spread the topic over all brokers.
 * @param zkUrl zookeeper connection url
 * @param topic topic name
 * @param replicationFactor the replication factor for the topic
 * @param partitionToBrokerRatio This is multiplied by the number brokers to compute the number of partitions in the topic.
 * @param topicConfig additional parameters for the topic for example min.insync.replicas
 * @return the number of partitions created
 */
public static int createMonitoringTopicIfNotExists(String zkUrl, String topic, int replicationFactor,
    double partitionToBrokerRatio, Properties topicConfig) {
  ZkUtils zkUtils = ZkUtils.apply(zkUrl, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, JaasUtils.isZkSecurityEnabled());
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      return getPartitionNumForTopic(zkUrl, topic);
    }
    int brokerCount = zkUtils.getAllBrokersInCluster().size();
    int partitionCount = (int) Math.ceil(brokerCount * partitionToBrokerRatio);

    try {
      AdminUtils.createTopic(zkUtils, topic, partitionCount, replicationFactor, topicConfig, RackAwareMode.Enforced$.MODULE$);
    } catch (TopicExistsException e) {
      //There is a race condition with the consumer.
      LOG.debug("Monitoring topic " + topic + " already exists in cluster " + zkUrl, e);
      return getPartitionNumForTopic(zkUrl, topic);
    }
    LOG.info("Created monitoring topic " + topic + " in cluster " + zkUrl + " with " + partitionCount + " partitions, min ISR of "
      + topicConfig.get(KafkaConfig.MinInSyncReplicasProp()) + " and replication factor of " + replicationFactor + ".");

    return partitionCount;
  } finally {
    zkUtils.close();
  }
}
 
Example 14
Project: HeliosStreams   File: KafkaAdminClient.java   View source code 6 votes vote down vote up
/**
 * Deletes the named topics
 * @param topicNames The names of the topics to delete
 * @return A set of the names of the topics that were successfully deleted
 */
public String[] deleteTopics(final String...topicNames) {
	if(!connected.get()) throw new IllegalStateException("The KafkaTestServer is not running");
	if(topicNames==null || topicNames.length==0) return new String[0];
	final Set<String> deleted = new LinkedHashSet<String>();
	for(String topicName: topicNames) {
		if(topicName==null || topicName.trim().isEmpty()) {
			try {
				AdminUtils.deleteTopic(zkUtils, topicName.trim());
				deleted.add(topicName.trim());
			} catch (Exception ex) {
				log.warn("Failed to delete topic [" + topicName.trim() + "]", ex);
			}
		}
	}
	return deleted.toArray(new String[deleted.size()]);
}
 
Example 15
Project: HeliosStreams   File: KafkaTestServer.java   View source code 6 votes vote down vote up
/**
 * Deletes the named topics
 * @param topicNames The names of the topics to delete
 * @return A set of the names of the topics that were successfully deleted
 */
public String[] deleteTopics(final String...topicNames) {
	if(!running.get()) throw new IllegalStateException("The KafkaTestServer is not running");
	if(topicNames==null || topicNames.length==0) return new String[0];
	final Set<String> deleted = new LinkedHashSet<String>();
	final ZkUtils z = getZkUtils();
	for(String topicName: topicNames) {
		if(topicName==null || topicName.trim().isEmpty()) {
			try {
				AdminUtils.deleteTopic(z, topicName.trim());
				deleted.add(topicName.trim());
			} catch (Exception ex) {
				log.warn("Failed to delete topic [" + topicName.trim() + "]", ex);
			}
		}
	}
	return deleted.toArray(new String[deleted.size()]);
}
 
Example 16
Project: incubator-pirk   File: KafkaStormIntegrationTest.java   View source code 6 votes vote down vote up
private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}
 
Example 17
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata();
			firstPart = partitionMetadata.head();
		}
		while (firstPart.errorCode() != 0);

		return firstPart.leader().get().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 18
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 19
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkClient zkClient = createZkClient();
	PartitionMetadata firstPart = null;
	do {
		if (firstPart != null) {
			LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
			// not the first try. Sleep a bit
			Thread.sleep(150);
		}

		Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
		firstPart = partitionMetadata.head();
	}
	while (firstPart.errorCode() != 0);
	zkClient.close();

	return firstPart.leader().get().id();
}
 
Example 20
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		MetadataResponse.PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.error().code());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
			firstPart = partitionMetadata.get(0);
		}
		while (firstPart.error().code() != 0);

		return firstPart.leader().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 21
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 22
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		MetadataResponse.PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.error().code());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
			firstPart = partitionMetadata.get(0);
		}
		while (firstPart.error().code() != 0);

		return firstPart.leader().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 23
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 24
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionsMetadata();
			firstPart = partitionMetadata.head();
		}
		while (firstPart.errorCode() != 0);

		return firstPart.leader().get().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 25
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 26
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkClient zkClient = createZkClient();
	PartitionMetadata firstPart = null;
	do {
		if (firstPart != null) {
			LOG.info("Unable to find leader. error code {}", firstPart.errorCode());
			// not the first try. Sleep a bit
			Thread.sleep(150);
		}

		Seq<PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkClient).partitionsMetadata();
		firstPart = partitionMetadata.head();
	}
	while (firstPart.errorCode() != 0);
	zkClient.close();

	return firstPart.leader().get().id();
}
 
Example 27
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public int getLeaderToShutDown(String topic) throws Exception {
	ZkUtils zkUtils = getZkUtils();
	try {
		MetadataResponse.PartitionMetadata firstPart = null;
		do {
			if (firstPart != null) {
				LOG.info("Unable to find leader. error code {}", firstPart.error().code());
				// not the first try. Sleep a bit
				Thread.sleep(150);
			}

			List<MetadataResponse.PartitionMetadata> partitionMetadata = AdminUtils.fetchTopicMetadataFromZk(topic, zkUtils).partitionMetadata();
			firstPart = partitionMetadata.get(0);
		}
		while (firstPart.error().code() != 0);

		return firstPart.leader().id();
	} finally {
		zkUtils.close();
	}
}
 
Example 28
Project: flink   File: KafkaTestEnvironmentImpl.java   View source code 6 votes vote down vote up
@Override
public void deleteTestTopic(String topic) {
	ZkUtils zkUtils = getZkUtils();
	try {
		LOG.info("Deleting topic {}", topic);

		ZkClient zk = new ZkClient(zookeeperConnectionString, Integer.valueOf(standardProps.getProperty("zookeeper.session.timeout.ms")),
			Integer.valueOf(standardProps.getProperty("zookeeper.connection.timeout.ms")), new ZooKeeperStringSerializer());

		AdminUtils.deleteTopic(zkUtils, topic);

		zk.close();
	} finally {
		zkUtils.close();
	}
}
 
Example 29
Project: drill   File: TestKafkaSuit.java   View source code 6 votes vote down vote up
@BeforeClass
public static void initKafka() throws Exception {
  synchronized (TestKafkaSuit.class) {
    if (initCount.get() == 0) {
      ZookeeperTestUtil.setZookeeperSaslTestConfigProps();
      System.setProperty(JaasUtils.JAVA_LOGIN_CONFIG_PARAM, ClassLoader.getSystemResource(LOGIN_CONF_RESOURCE_PATHNAME).getFile());
      embeddedKafkaCluster = new EmbeddedKafkaCluster();
      Properties topicProps = new Properties();
      zkClient = new ZkClient(embeddedKafkaCluster.getZkServer().getConnectionString(), SESSION_TIMEOUT, CONN_TIMEOUT, ZKStringSerializer$.MODULE$);
      ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(embeddedKafkaCluster.getZkServer().getConnectionString()), false);
      AdminUtils.createTopic(zkUtils, QueryConstants.JSON_TOPIC, 1, 1, topicProps, RackAwareMode.Disabled$.MODULE$);

      org.apache.kafka.common.requests.MetadataResponse.TopicMetadata fetchTopicMetadataFromZk = AdminUtils
          .fetchTopicMetadataFromZk(QueryConstants.JSON_TOPIC, zkUtils);
      logger.info("Topic Metadata: " + fetchTopicMetadataFromZk);

      KafkaMessageGenerator generator = new KafkaMessageGenerator(embeddedKafkaCluster.getKafkaBrokerList(),
          StringSerializer.class);
      generator.populateJsonMsgIntoKafka(QueryConstants.JSON_TOPIC, NUM_JSON_MSG);
    }
    initCount.incrementAndGet();
    runningSuite = true;
  }
  logger.info("Initialized Embedded Zookeeper and Kafka");
}
 
Example 30
Project: ksql   File: KafkaEmbedded.java   View source code 6 votes vote down vote up
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic,
                        int partitions,
                        int replication,
                        Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);
  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // registerTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  boolean isSecure = false;
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), isSecure);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 
Example 31
Project: incubator-rya   File: KafkaTopics.java   View source code 6 votes vote down vote up
/**
 * Creates a set of Kafka topics for each topic that does not already exist.
 *
 * @param zookeeperServers - The Zookeeper servers that are used by the Kafka Streams program. (not null)
 * @param topicNames - The topics that will be created. (not null)
 * @param partitions - The number of partitions that each of the topics will have.
 * @param replicationFactor - The replication factor of the topics that are created.
 */
public static void createTopic(
        final String zookeeperServers,
        final Set<String> topicNames,
        final int partitions,
        final int replicationFactor) {
    requireNonNull(zookeeperServers);
    requireNonNull(topicNames);

    ZkUtils zkUtils = null;
    try {
        zkUtils = ZkUtils.apply(new ZkClient(zookeeperServers, 30000, 30000, ZKStringSerializer$.MODULE$), false);
        for(final String topicName : topicNames) {
            if(!AdminUtils.topicExists(zkUtils, topicName)) {
                AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, new Properties(), RackAwareMode.Disabled$.MODULE$);
            }
        }
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 
Example 32
Project: ameliant-tools   File: EmbeddedKafkaBroker.java   View source code 6 votes vote down vote up
@Override
protected void before() throws Throwable {
    logDirectory = tempDir(perTest("kafka-log"));
    Properties properties = brokerDefinition.getProperties();
    properties.setProperty(KafkaConfig.LogDirProp(), logDirectory.getCanonicalPath());
    kafkaServer = new KafkaServer(new KafkaConfig(properties),
            SystemTime$.MODULE$, Some$.MODULE$.apply("kafkaServer"));
    kafkaServer.startup();

    List<TopicDefinition> topicDefinitions = brokerDefinition.getTopicDefinitions();
    if (!topicDefinitions.isEmpty()) {
        ZkUtils zkUtils = ZkUtils.apply(brokerDefinition.getZookeeperConnect(), 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        for (TopicDefinition topicDefinition : topicDefinitions) {
            String name = topicDefinition.getName();
            log.info("Creating topic {}", name);
            AdminUtils.createTopic(zkUtils,
                    name,
                    topicDefinition.getPartitions(),
                    topicDefinition.getReplicationFactor(),
                    topicDefinition.getProperties());
        }
    }
}
 
Example 33
Project: kite-apps   File: Topics.java   View source code 6 votes vote down vote up
public static void createTopic(AppContext context,
                               String topicName,
                               int partitions,
                               int replicationFactor,
                               Schema schema) {

  // Currently the schema parameter is unused, but we have it here so it can
  // be applied to a schema registry for the topic when that is available.

  ZkClient client = new ZkClient(getKafkaProp(context, ZOOKEEPER_CONNECT), 1000, 1000, ZKStringSerializer$.MODULE$);

  try {
    if (!AdminUtils.topicExists(client, topicName)) {
      AdminUtils.createTopic(client, topicName, partitions, replicationFactor, new Properties());
    }
  } finally {
    client.close();
  }
}
 
Example 34
Project: iotanalytics-backend   File: KafkaSenderServiceTest.java   View source code 6 votes vote down vote up
@Before
public void initMocks() throws Exception {
    MockitoAnnotations.initMocks(this);

    PowerMockito.whenNew(ZkClient.class).withAnyArguments().thenReturn(zkClient);
    Mockito.doNothing().when(zkClient).close();

    PowerMockito.mockStatic(AdminUtils.class);
    Mockito.when(AdminUtils.topicExists(zkClient, TOPIC)).thenReturn(true);

    Mockito.when(serviceConfigProvider.getKafkaTopicName()).thenReturn(TOPIC);
    Mockito.when(serviceConfigProvider.getKafkaPartitionsFactor()).thenReturn(1);
    Mockito.when(serviceConfigProvider.getKafkaReplicationFactor()).thenReturn(1);
    Mockito.when(serviceConfigProvider.getKafkaTimeoutInMs()).thenReturn(10);
    Mockito.when(serviceConfigProvider.getZookeeperUri()).thenReturn("localhost");
}
 
Example 35
Project: cloudera-framework   File: KafkaServer.java   View source code 6 votes vote down vote up
/**
 * Create topic
 *
 * @return if the topic was created or not
 */
public synchronized boolean createTopic(String topic, int partitions, int replicationFactor, Properties properties)
  throws InterruptedException {
  boolean created = true;
  try {
    AdminUtils.createTopic(getZooKeeperUtils(), topic, partitions, replicationFactor, properties, AdminUtils.createTopic$default$6());
  } catch (TopicExistsException topicExistsException) {
    created = false;
  }
  while (AdminUtils.fetchTopicMetadataFromZk(topic, zooKeeperUtils).toString().contains("LeaderNotAvailableException")) {
    if (LOG.isTraceEnabled()) {
      LOG.trace("Sleeping for [" + KAFKA_POLL_MS + "] ms, waiting for Kafka topic to be registered in ZooKeeper");
    }
    Thread.sleep(KAFKA_POLL_MS);
  }
  return created;
}
 
Example 36
Project: producer   File: MessageService.java   View source code 6 votes vote down vote up
public void createTopic(String topic) {
    log.debug("Create Topic {}.", topic);
    if (doesTopicExist(topic)) {
        log.debug("Topic %s already exists.", topic);
        return;
    }
     // Create a ZooKeeper client
    int sessionTimeoutMs = 10000;
    int connectionTimeoutMs = 10000;
    ZkClient zkClient = new ZkClient("localhost:2181", sessionTimeoutMs, connectionTimeoutMs, ZKStringSerializer$.MODULE$);

    // Create the topic
    int numPartitions = 1;
    int replicationFactor = 1;
    Properties topicConfig = new Properties();
    AdminUtils.createTopic(zkClient, topic, numPartitions, replicationFactor, topicConfig);
    log.debug("Topic {} is created.", topic);
}
 
Example 37
Project: product-cep   File: KafkaTestCase.java   View source code 6 votes vote down vote up
private void setupKafkaBroker() {
    try {
        // mock zookeeper
        zkTestServer = new TestingServer(2181);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "0");
        props.put("host.name", "localhost");
        props.put("port", "9092");
        props.put("log.dir", "/tmp/tmp_kafka_dir");
        props.put("zookeeper.connect", zkTestServer.getConnectString());
        props.put("replica.socket.timeout.ms", "1500");
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer = new KafkaServerStartable(config);
        kafkaServer.startup();

        // create "sensordata" topic
        ZkClient zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);
        AdminUtils.createTopic(zkClient, "sensordata", 1, 1, new Properties());
        zkClient.close();
    } catch (Exception e) {
        log.error("Error running local Kafka broker / Zookeeper", e);
    }
}
 
Example 38
Project: devicehive-java-server   File: KafkaRpcTopicService.java   View source code 6 votes vote down vote up
public void createTopic(String topic) {
    ZkClient zkClient = new ZkClient(
            kafkaRpcConfig.getZookeeperConnect(),
            kafkaRpcConfig.getSessionTimeout(),
            kafkaRpcConfig.getConnectionTimeout(),
            ZKStringSerializer$.MODULE$);
    try {
        ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(kafkaRpcConfig.getZookeeperConnect()), false);
        Properties topicConfig = kafkaRpcConfig.topicProps();
        if (!AdminUtils.topicExists(zkUtils, topic)) {
            AdminUtils.createTopic(zkUtils, topic, kafkaRpcConfig.getNumPartitions(), 
                    kafkaRpcConfig.getReplicationFactor(), topicConfig, RackAwareMode.Enforced$.MODULE$);
        }
    } finally {
        zkClient.close();
    }
}
 
Example 39
Project: management-sdk-for-kafka   File: ClusterTools.java   View source code 5 votes vote down vote up
/**
 * Override topic configuration
 *
 * @param connection zookeeper util API
 * @param topicName  topic name
 * @param configs    topic properties
 */
public void overrideTopicProperties(final ZkUtils connection, final String topicName, final Properties configs) {
    try {
        AdminUtils.changeTopicConfig(connection, topicName, configs);
    } catch (AdminOperationException | KafkaException e) {
        throw new TopicOperationException(topicName, e.getMessage(), e, this.getClass());
    }
}
 
Example 40
Project: cruise-control   File: LoadMonitorTaskRunnerTest.java   View source code 5 votes vote down vote up
@Before
public void setUp() {
  super.setUp();
  ZkUtils zkUtils = CruiseControlUnitTestUtils.zkUtils(zookeeper().getConnectionString());
  for (int i = 0; i < NUM_TOPICS; i++) {
    AdminUtils.createTopic(zkUtils, "topic-" + i, NUM_PARTITIONS, 1, new Properties(), RackAwareMode.Safe$.MODULE$);
  }
}