Java Code Examples for kafka.javaapi.PartitionMetadata#leader()

The following examples show how to use kafka.javaapi.PartitionMetadata#leader() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: KafkaLowLevelConsumer09.java From datacollector with Apache License 2.0

6 votes

private HostAndPort findNewLeader(HostAndPort oldLeader, String topic, int partition) throws StageException {
  //try 3 times to find a new leader
  for (int i = 0; i < 3; i++) {
    boolean sleep;
    PartitionMetadata metadata = getPartitionMetadata(replicaBrokers, topic, partition);
    if (metadata == null || metadata.leader() == null) {
      sleep = true;
    } else if (oldLeader.getHostText().equalsIgnoreCase(metadata.leader().host()) && i == 0) {
      //leader has not yet changed, give zookeeper sometime
      sleep = true;
    } else {
      return HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
    }
    if (sleep) {
      ThreadUtil.sleep(ONE_SECOND);
    }
  }
  LOG.error(KafkaErrors.KAFKA_21.getMessage());
  throw new StageException(KafkaErrors.KAFKA_21);
}

Example 2

Source File: AckKafkaWriterTestRunner.java From Scribengin with GNU Affero General Public License v3.0

6 votes

public void run() {
  try {
    while (!exit) {
      KafkaTool kafkaTool = new KafkaTool(topic, cluster.getZKConnect());
      kafkaTool.connect();
      TopicMetadata topicMeta = kafkaTool.findTopicMetadata(topic);
      PartitionMetadata partitionMeta = findPartition(topicMeta, partition);
      Broker partitionLeader = partitionMeta.leader();
      Server kafkaServer = cluster.findKafkaServerByPort(partitionLeader.port());
      System.out.println("Shutdown kafka server " + kafkaServer.getPort());
      kafkaServer.shutdown();
      failureCount++;
      Thread.sleep(sleepBeforeRestart);
      kafkaServer.start();
      kafkaTool.close();
      Thread.sleep(10000); //wait to make sure that the kafka server start
    }
  } catch (Exception e) {
    e.printStackTrace();
  }
  synchronized (this) {
    notify();
  }
}

Example 3

Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0

6 votes

private HostPort findNewLeader(String oldHost, int oldPort) throws LostLeadershipException {
  for (int i = 0; i < 3; i++) {
    boolean goToSleep = false;
    PartitionMetadata metadata = findLeader(replicaBrokers, topic, partition);
    if (metadata == null) {
      goToSleep = true;
    } else if (metadata.leader() == null) {
      goToSleep = true;
    } else if (oldHost.equalsIgnoreCase(metadata.leader().host()) &&
        oldPort == metadata.leader().port()) {
      // first time through if the leader hasn't changed give ZooKeeper a second to recover
      // second time, assume the broker did recover before failover, or it was a non-Broker issue
      goToSleep = true;
    } else {
      return new HostPort(metadata.leader().host(), metadata.leader().port());
    }
    if (goToSleep) {
      try {
        Thread.sleep(1000);
      } catch (InterruptedException ie) {
      }
    }
  }
  // Can't recover from a leadership disappearance.
  throw new LostLeadershipException();
}

Example 4

Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0

6 votes

private List<KafkaPartition> getPartitionsForTopic(TopicMetadata topicMetadata) {
  List<KafkaPartition> partitions = Lists.newArrayList();

  for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
    if (null == partitionMetadata) {
      log.error("Ignoring topic with null partition metadata " + topicMetadata.topic());
      return Collections.emptyList();
    }
    if (null == partitionMetadata.leader()) {
      log.error("Ignoring topic with null partition leader " + topicMetadata.topic() + " metatada="
          + partitionMetadata);
      return Collections.emptyList();
    }
    partitions.add(new KafkaPartition.Builder().withId(partitionMetadata.partitionId())
        .withTopicName(topicMetadata.topic()).withLeaderId(partitionMetadata.leader().id())
        .withLeaderHostAndPort(partitionMetadata.leader().host(), partitionMetadata.leader().port()).build());
  }
  return partitions;
}

Example 5

Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0

6 votes

private List<KafkaPartition> getPartitionsForTopic(TopicMetadata topicMetadata) {
  List<KafkaPartition> partitions = Lists.newArrayList();

  for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) {
    if (null == partitionMetadata) {
      LOG.error("Ignoring topic with null partition metadata " + topicMetadata.topic());
      return Collections.emptyList();
    }
    if (null == partitionMetadata.leader()) {
      LOG.error(
          "Ignoring topic with null partition leader " + topicMetadata.topic() + " metatada=" + partitionMetadata);
      return Collections.emptyList();
    }
    partitions.add(new KafkaPartition.Builder().withId(partitionMetadata.partitionId())
        .withTopicName(topicMetadata.topic()).withLeaderId(partitionMetadata.leader().id())
        .withLeaderHostAndPort(partitionMetadata.leader().host(), partitionMetadata.leader().port()).build());
  }
  return partitions;
}

Example 6

Source File: NativeSimpleConsumer.java From spring-kafka-demo with Apache License 2.0

6 votes

private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
            // first time through if the leader hasn't changed give ZooKeeper a second to recover
            // second time, assume the broker did recover before failover, or it was a non-Broker issue
            //
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    System.out.println("Unable to find new leader after Broker failure. Exiting");
    throw new Exception("Unable to find new leader after Broker failure. Exiting");
}

Example 7

Source File: KafkaSimpleConsumer.java From Pistachio with Apache License 2.0

6 votes

private Broker findNewLeader(Broker oldLeader) throws InterruptedException {
    long retryCnt = 0;
    while (true) {
        PartitionMetadata metadata = findLeader();
        logger.debug("findNewLeader - meta leader {}, previous leader {}", metadata, oldLeader);
        if (metadata != null && metadata.leader() != null && (oldLeader == null ||
                (!(oldLeader.host().equalsIgnoreCase(metadata.leader().host()) &&
                  (oldLeader.port() == metadata.leader().port())) || retryCnt != 0))) {
            // first time through if the leader hasn't changed give ZooKeeper a second to recover
            // second time, assume the broker did recover before failover, or it was a non-Broker issue
            logger.info("findNewLeader - using new leader {} from meta data, previous leader {}", metadata.leader(), oldLeader);
            return metadata.leader();
        }
        //TODO: backoff retry
        Thread.sleep(1000L);
        retryCnt ++;
        // if could not find the leader for current replicaBrokers, let's try to find one via allBrokers
        if (retryCnt >= 3 && (retryCnt - 3) % 5 == 0) {
            logger.warn("can nof find leader for {} - {} after {} retries", topic, partitionId, retryCnt);
            replicaBrokers.clear();
            replicaBrokers.addAll(allBrokers);
        }
    }
}

Example 8

Source File: SimpleKafkaMessageListener.java From micro-integrator with Apache License 2.0

6 votes

public boolean run() throws Exception {

        if (init) {
            return init;
        }
        // find the meta data about the topic and partition we are interested in
        PartitionMetadata metadata = findLeader(seedBrokers, port, topic, partition);
        if (metadata == null) {
            throw new SynapseException("Can't find metadata for Topic and Partition. Exiting");
        }
        if (metadata.leader() == null) {
            throw new SynapseException("Can't find Leader for Topic and Partition. Exiting");
        }
        this.leadBroker = metadata.leader().host();
        this.clientName = "Client_" + topic + "_" + partition;

        this.consumer = new SimpleConsumer(leadBroker, port, KAFKAConstants.BUFFER_SIZE, KAFKAConstants.SO_TIMEOUT,
                                           clientName);
        this.readOffset = getLastOffset(consumer, topic, partition, kafka.api.OffsetRequest.EarliestTime(), clientName);
        init = true;

        return init;
    }

Example 9

Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0

6 votes

private HostAndPort findNewLeader(HostAndPort oldLeader, String topic, int partition) throws StageException {
  //try 3 times to find a new leader
  for (int i = 0; i < 3; i++) {
    boolean sleep;
    PartitionMetadata metadata = getPartitionMetadata(replicaBrokers, topic, partition);
    if (metadata == null || metadata.leader() == null) {
      sleep = true;
    } else if (oldLeader.getHostText().equalsIgnoreCase(metadata.leader().host()) && i == 0) {
      //leader has not yet changed, give zookeeper sometime
      sleep = true;
    } else {
      return HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
    }
    if (sleep) {
      ThreadUtil.sleep(ONE_SECOND);
    }
  }
  LOG.error(KafkaErrors.KAFKA_21.getMessage());
  throw new StageException(KafkaErrors.KAFKA_21);
}

Example 10

Source File: KafkaTestConsumer.java From hadoop-mini-clusters with Apache License 2.0

6 votes

private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
            // first time through if the leader hasn't changed give ZooKeeper a second to recover
            // second time, assume the broker did recover before failover, or it was a non-Broker issue
            //
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    LOG.info("Unable to find new leader after Broker failure. Exiting");
    throw new Exception("Unable to find new leader after Broker failure. Exiting");
}

Example 11

Source File: KafkaLatestOffsetFetcher.java From eagle with Apache License 2.0

6 votes

public Map<Integer, Long> fetch(String topic, int partitionCount) {
    Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount);
    Map<Integer, Long> ret = new HashMap<>();
    for (int partition = 0; partition < partitionCount; partition++) {
        PartitionMetadata metadata = metadatas.get(partition);
        if (metadata == null || metadata.leader() == null) {
            ret.put(partition, -1L);
            //throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting");
        }
        String leadBroker = metadata.leader().host();
        String clientName = "Client_" + topic + "_" + partition;
        SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);
        long latestOffset = getLatestOffset(consumer, topic, partition, clientName);
        if (consumer != null) {
            consumer.close();
        }
        ret.put(partition, latestOffset);
    }
    return ret;
}

Example 12

Source File: SimpleKafkaMessageListener.java From micro-integrator with Apache License 2.0

6 votes

private String findNewLeader(String oldLeader, String topic, int partition, int port) throws Exception {
    for (int i = 0; i < 3; i++) {
        boolean goToSleep = false;
        PartitionMetadata metadata = findLeader(replicaBrokers, port, topic, partition);
        if (metadata == null) {
            goToSleep = true;
        } else if (metadata.leader() == null) {
            goToSleep = true;
        } else if (oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
            goToSleep = true;
        } else {
            return metadata.leader().host();
        }
        if (goToSleep) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ie) {
            }
        }
    }
    throw new SynapseException("Unable to find new leader after Broker failure. Exiting");
}

Example 13

Source File: KafkaLowLevelConsumer09.java From datacollector with Apache License 2.0

5 votes

@Override
public void init() throws StageException {
  List<HostAndPort> brokers = new ArrayList<>();
  brokers.add(broker);
  PartitionMetadata metadata = getPartitionMetadata(brokers, topic, partition);
  if (metadata == null) {
    LOG.error(KafkaErrors.KAFKA_23.getMessage(), topic, partition);
    throw new StageException(KafkaErrors.KAFKA_23, topic, partition);
  }
  if (metadata.leader() == null) {
    LOG.error(KafkaErrors.KAFKA_24.getMessage(), topic, partition);
    throw new StageException(KafkaErrors.KAFKA_24, topic, partition);
  }
  leader = HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
  //recreate consumer instance with the leader information for that topic
  LOG.info(
      "Creating SimpleConsumer using the following configuration: host {}, port {}, max wait time {}, max " +
      "fetch size {}, client columnName {}",
      leader.getHostText(),
      leader.getPort(),
      maxWaitTime,
      maxFetchSize,
      clientName
  );
  consumer = new SimpleConsumer(
      leader.getHostText(),
      leader.getPort(),
      maxWaitTime,
      maxFetchSize,
      clientName
  );
}

Example 14

Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0

5 votes

@Override
public void init() throws StageException {
  List<HostAndPort> brokers = new ArrayList<>();
  brokers.add(broker);
  PartitionMetadata metadata = getPartitionMetadata(brokers, topic, partition);
  if (metadata == null) {
    LOG.error(KafkaErrors.KAFKA_23.getMessage(), topic, partition);
    throw new StageException(KafkaErrors.KAFKA_23, topic, partition);
  }
  if (metadata.leader() == null) {
    LOG.error(KafkaErrors.KAFKA_24.getMessage(), topic, partition);
    throw new StageException(KafkaErrors.KAFKA_24, topic, partition);
  }
  leader = HostAndPort.fromParts(metadata.leader().host(), metadata.leader().port());
  //recreate consumer instance with the leader information for that topic
  LOG.info(
      "Creating SimpleConsumer using the following configuration: host {}, port {}, max wait time {}, max " +
      "fetch size {}, client columnName {}",
      leader.getHostText(),
      leader.getPort(),
      maxWaitTime,
      maxFetchSize,
      clientName
  );
  consumer = new SimpleConsumer(
      leader.getHostText(),
      leader.getPort(),
      maxWaitTime,
      maxFetchSize,
      clientName
  );
}

Example 15

Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0

5 votes

public boolean connectToTopic() {
  boolean r = true;
  PartitionMetadata metadata = findLeader(brokerList, topic, partition);
  if (metadata == null) {
    r = false;
    LOG.error("Can't find meta data for Topic: " + topic + " partition: " + partition
        + ". In fact, meta is null.");
  }
  else if (metadata.leader() == null) {
    r = false;
    LOG.error("Can't find meta data for Topic: " + topic + " partition: " + partition);
  }

  if (r) {
    storeReplicaBrokers(metadata);

    consumer = new SimpleConsumer(
        metadata.leader().host(),
        metadata.leader().port(),
        10000, // timeout
        64 * 1024, // buffersize
        getClientName());

    //scheduleCommitTimer();
  }
  return r;
}

Example 16

Source File: SimpleConsumerExample.java From hermes with Apache License 2.0

5 votes

/**
 * @param a_oldLeader
 * @param a_topic
 * @param a_partition
 * @param a_port
 * @return String
 * @throws Exception
 *            找一个leader broker
 */
private String findNewLeader(String a_oldLeader, String a_topic, int a_partition, int a_port) throws Exception {
	for (int i = 0; i < 3; i++) {
		boolean goToSleep = false;
		PartitionMetadata metadata = findLeader(m_replicaBrokers, a_port, a_topic, a_partition);
		if (metadata == null) {
			goToSleep = true;
		} else if (metadata.leader() == null) {
			goToSleep = true;
		} else if (a_oldLeader.equalsIgnoreCase(metadata.leader().host()) && i == 0) {
			// first time through if the leader hasn't changed give
			// ZooKeeper a second to recover
			// second time, assume the broker did recover before failover,
			// or it was a non-Broker issue
			//
			goToSleep = true;
		} else {
			return metadata.leader().host();
		}
		if (goToSleep) {
			try {
				Thread.sleep(1000);
			} catch (InterruptedException ie) {
			}
		}
	}
	System.out.println("Unable to find new leader after Broker failure. Exiting");
	throw new Exception("Unable to find new leader after Broker failure. Exiting");
}

Example 17

Source File: KafkaClusterTool.java From Scribengin with GNU Affero General Public License v3.0

5 votes

Server findLeader(String topic, int partition) throws Exception {
  KafkaTool kafkaTool = new KafkaTool("KafkaPartitionLeaderKiller", cluster.getZKConnect());
  kafkaTool.connect();
  TopicMetadata topicMeta = kafkaTool.findTopicMetadata(topic);
  PartitionMetadata partitionMeta = findPartition(topicMeta, partition);
  Broker partitionLeader = partitionMeta.leader();
  Server kafkaServer = cluster.findKafkaServerByPort(partitionLeader.port());
  System.out.println("Shutdown kafka server " + kafkaServer.getPort());
  kafkaTool.close();
  return kafkaServer;
}