Java Code Examples for kafka.zk.KafkaZkClient#apply()

The following examples show how to use kafka.zk.KafkaZkClient#apply() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: KafkaUnit.java    From SkaETL with Apache License 2.0 7 votes vote down vote up 
public void createTopic(String topicName, int numPartitions) {
    // setup

    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: CreateTopic " + topicName);
        adminZkClient.createTopic(topicName,numPartitions, 1,new Properties(), RackAwareMode.Disabled$.MODULE$);
    } finally {
        zkClient.close();
    }

}
Example 2
Source File: KafkaUnit.java    From SkaETL with Apache License 2.0 6 votes vote down vote up 
/**
 * @return All topic names
 */
public List<String> listTopics() {
    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: ListTopics ");

            return JavaConversions.asJavaCollection(adminZkClient.getAllTopicConfigs().keys())
                    .stream()
                    .collect(Collectors.toList());

    } finally {
        zkClient.close();
    }
}
Example 3
Source File: KafkaUnit.java    From SkaETL with Apache License 2.0 6 votes vote down vote up 
/**
 * Delete a topic.
 *
 * @param topicName The name of the topic to delete
 */
public void deleteTopic(String topicName) {
    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: DeleteTopic " + topicName);
        adminZkClient.deleteTopic(topicName);
    } finally {
        zkClient.close();
    }
}
Example 4
Source File: MultiClusterTopicManagementService.java    From kafka-monitor with Apache License 2.0 6 votes vote down vote up 
void maybeElectLeader() throws Exception {
  if (!_preferredLeaderElectionRequested) {
    return;
  }

  try (KafkaZkClient zkClient = KafkaZkClient.apply(_zkConnect, JaasUtils.isZkSecurityEnabled(), com.linkedin.kmf.common.Utils.ZK_SESSION_TIMEOUT_MS,
      com.linkedin.kmf.common.Utils.ZK_CONNECTION_TIMEOUT_MS, Integer.MAX_VALUE, Time.SYSTEM, METRIC_GROUP_NAME, "SessionExpireListener", null)) {
    if (!zkClient.reassignPartitionsInProgress()) {
      List<TopicPartitionInfo> partitionInfoList = _adminClient
          .describeTopics(Collections.singleton(_topic)).all().get().get(_topic).partitions();
      LOGGER.info(
          "MultiClusterTopicManagementService will trigger requested preferred leader election for the"
              + " topic {} in cluster.", _topic);
      triggerPreferredLeaderElection(partitionInfoList, _topic);
      _preferredLeaderElectionRequested = false;
    }
  }
}
Example 5
Source File: KafkaStarterUtils.java    From uReplicator with Apache License 2.0 5 votes vote down vote up 
public static void createTopic(String kafkaTopic, int numOfPartitions, String zkStr, String replicatorFactor) {
  // TopicCommand.main() will call System.exit() finally, which will break maven-surefire-plugin
  try {
    String[] args = new String[]{"--create", "--zookeeper", zkStr, "--replication-factor", replicatorFactor,
        "--partitions", String.valueOf(numOfPartitions), "--topic", kafkaTopic};
    KafkaZkClient zkClient = KafkaZkClient
        .apply(zkStr, false, 3000, 3000, Integer.MAX_VALUE, Time.SYSTEM, "kafka.server",
            "SessionExpireListener");
    TopicCommand.TopicCommandOptions opts = new TopicCommand.TopicCommandOptions(args);
    TopicCommand.createTopic(zkClient, opts);
  } catch (TopicExistsException e) {
    // Catch TopicExistsException otherwise it will break maven-surefire-plugin
    System.out.println("Topic already existed");
  }
}
Example 6
Source File: EmbeddedKafka.java    From mongo-kafka with Apache License 2.0 4 votes vote down vote up 
/** Creates and starts the cluster. */
public void start() throws Exception {
  LOGGER.debug("Initiating embedded Kafka cluster startup");
  LOGGER.debug("Starting a ZooKeeper instance...");
  zookeeper = new ZooKeeperEmbedded();
  LOGGER.debug("ZooKeeper instance is running at {}", zookeeper.connectString());

  zkClient =
      KafkaZkClient.apply(
          zookeeper.connectString(),
          JaasUtils.isZkSecurityEnabled(),
          30000,
          30000,
          1000,
          new MockTime(),
          "kafka.server",
          "SessionExpireListener");

  final Properties effectiveBrokerConfig = effectiveBrokerConfigFrom(brokerConfig, zookeeper);
  LOGGER.debug(
      "Starting a Kafka instance on port {} ...",
      effectiveBrokerConfig.getProperty(KafkaConfig$.MODULE$.PortProp()));
  broker = new KafkaEmbedded(effectiveBrokerConfig, new MockTime());
  LOGGER.debug(
      "Kafka instance is running at {}, connected to ZooKeeper at {}",
      broker.brokerList(),
      broker.zookeeperConnect());

  final Properties schemaRegistryProps = new Properties();
  schemaRegistryProps.put(
      SchemaRegistryConfig.KAFKASTORE_TIMEOUT_CONFIG, KAFKASTORE_OPERATION_TIMEOUT_MS);
  schemaRegistryProps.put(SchemaRegistryConfig.DEBUG_CONFIG, KAFKASTORE_DEBUG);
  schemaRegistryProps.put(
      SchemaRegistryConfig.KAFKASTORE_INIT_TIMEOUT_CONFIG, KAFKASTORE_INIT_TIMEOUT);

  schemaRegistry =
      new RestApp(
          0,
          zookeeperConnect(),
          KAFKA_SCHEMAS_TOPIC,
          AVRO_COMPATIBILITY_TYPE,
          schemaRegistryProps);
  schemaRegistry.start();

  LOGGER.debug("Starting a Connect standalone instance...");
  connect = new ConnectStandalone(connectWorkerConfig());
  connect.start();
  LOGGER.debug("Connect standalone instance is running at {}", connect.getConnectionString());
  running = true;
}
Example 7
Source File: KafkaCruiseControlUtils.java    From cruise-control with BSD 2-Clause "Simplified" License 2 votes vote down vote up 
/**
 * Create an instance of KafkaZkClient with security disabled.
 * Name of the underlying {@link kafka.zookeeper.ZooKeeperClient} instance is derived using the combination given
 * metricGroup and metricType with a dash in between.
 *
 * @param connectString Comma separated host:port pairs, each corresponding to a zk server
 * @param metricGroup Metric group
 * @param metricType Metric type
 * @param zkSecurityEnabled True if zkSecurityEnabled, false otherwise.
 * @return A new instance of KafkaZkClient
 */
public static KafkaZkClient createKafkaZkClient(String connectString, String metricGroup, String metricType, boolean zkSecurityEnabled) {
  String zooKeeperClientName = String.format("%s-%s", metricGroup, metricType);
  return KafkaZkClient.apply(connectString, zkSecurityEnabled, ZK_SESSION_TIMEOUT, ZK_CONNECTION_TIMEOUT, Integer.MAX_VALUE,
                             new SystemTime(), metricGroup, metricType, Option.apply(zooKeeperClientName));
}