Java Code Examples for org.apache.kafka.common.Cluster#partitionsForTopic()

/**
 * 决定消息被写入哪个分区
 * @param topic topic
 * @param key key
 * @param keyBytes key serialize byte
 * @param value value
 * @param valueBytes value serialize byte
 * @param cluster kakfa cluster
 * @return
 */
@Override
public int partition(String topic, Object key, byte[] keyBytes,
                     Object value, byte[] valueBytes, Cluster cluster) {
    // 所有分区信息
    List<PartitionInfo> partitionInfos = cluster.partitionsForTopic(topic);
    int partitionCount = partitionInfos.size();
    // 要求必须存在 key,如果key 是"name" 就分配到最后一个分区, 其他key hash取模
    if (keyBytes == null || !key.getClass().equals(String.class)) {
        throw new InvalidRecordException("kafka message must have a String key");
    }
    if (partitionCount == 1 || StringUtils.endsWithIgnoreCase("name", key.toString())) {
        return partitionCount - 1;
    }
    return Math.abs(Utils.murmur2(keyBytes)) % (partitionCount - 1);
}
 

public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
  List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
  int numPartitions = partitions.size();
  int msgCount = normalCounter.incrementAndGet();
  if (msgCount % 1000 == 0) {
    log.info("Sent {} messages", msgCount);
  }

  if (key == null) {
    int nextValue = this.counter.getAndIncrement();
    List<PartitionInfo> availablePartitions = cluster.availablePartitionsForTopic(topic);
    if (availablePartitions.size() > 0) {
      int part = toPositive(nextValue) % availablePartitions.size();
      return availablePartitions.get(part).partition();
    } else {
      return toPositive(nextValue) % numPartitions;
    }
  } else {
    return computePartition(key, numPartitions, maxParallelism);
  }
}
 

/**
 * Add the partitions info for the new topic.
 * Need to ensure the atomicity of the update operation.
 */
private synchronized Cluster addPartitionsInfo(Cluster cluster, String topic) {
    List<String> partitions = client.getPartitionsForTopic(topic).join();
    // read partitions info
    Set<PartitionInfo> partitionsInfo = new HashSet<>();
    for (int i = 0; i < partitions.size(); i++) {
        partitionsInfo.add(new PartitionInfo(topic, i, null, null, null));
    }
    // create cluster with new partitions info
    Set<PartitionInfo> combinedPartitions = new HashSet<>();
    if (cluster.partitionsForTopic(topic) != null) {
        combinedPartitions.addAll(cluster.partitionsForTopic(topic));
    }
    combinedPartitions.addAll(partitionsInfo);
    return new Cluster(cluster.nodes(), combinedPartitions, new HashSet(cluster.unauthorizedTopics()));
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
	List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
	int numPartitions = partitions.size();
	if (keyBytes != null) {
		int hashCode = 0;
		if (key instanceof Integer || key instanceof Long) {
			hashCode = (int) key;
		} else {
			hashCode = key.hashCode();
		}
		hashCode = hashCode & 0x7fffffff;
		return hashCode % numPartitions;
	} else {
		return 0;
	}
}
 

/**
 * Scan through topics to check whether the topic having partition(s) with bad replication factor. For each topic, the
 * target replication factor to check against is the maximum value of {@link #SELF_HEALING_TARGET_TOPIC_REPLICATION_FACTOR_CONFIG}
 * and topic's minISR plus value of {@link #TOPIC_REPLICATION_FACTOR_MARGIN_CONFIG}.
 *
 * @param topicsToCheck Set of topics to check.
 * @return Map of detected topic replication factor anomaly entries by target replication factor.
 */
private Map<Short, Set<TopicReplicationFactorAnomalyEntry>> populateBadTopicsByReplicationFactor(Set<String> topicsToCheck, Cluster cluster) {
  Map<Short, Set<TopicReplicationFactorAnomalyEntry>> topicsByReplicationFactor = new HashMap<>();
  for (String topic : topicsToCheck) {
    if (_cachedTopicMinISR.containsKey(topic)) {
      short topicMinISR = _cachedTopicMinISR.get(topic).minISR();
      short targetReplicationFactor = (short) Math.max(_targetReplicationFactor, topicMinISR + _topicReplicationFactorMargin);
      int violatedPartitionCount = 0;
      for (PartitionInfo partitionInfo : cluster.partitionsForTopic(topic)) {
        if (partitionInfo.replicas().length != targetReplicationFactor) {
          violatedPartitionCount++;
        }
      }
      if (violatedPartitionCount > 0) {
        topicsByReplicationFactor.putIfAbsent(targetReplicationFactor, new HashSet<>());
        topicsByReplicationFactor.get(targetReplicationFactor).add(
            new TopicReplicationFactorAnomalyEntry(topic, (double) violatedPartitionCount /  cluster.partitionCountForTopic(topic)));
      }
    }
  }
  return topicsByReplicationFactor;
}
 

@Override
public Set<TopicAnomaly> topicAnomalies() {
  LOG.info("Start to detect topic replication factor anomaly.");
  Cluster cluster = _kafkaCruiseControl.kafkaCluster();
  Set<String> topicsToCheck = new HashSet<>();
  for (String topic : cluster.topics()) {
    if (_topicExcludedFromCheck.matcher(topic).matches()) {
      continue;
    }
    for (PartitionInfo partition : cluster.partitionsForTopic(topic)) {
      if (partition.replicas().length != _targetReplicationFactor) {
        topicsToCheck.add(topic);
        break;
      }
    }
  }
  refreshTopicMinISRCache();
  if (!topicsToCheck.isEmpty()) {
    maybeRetrieveAndCacheTopicMinISR(topicsToCheck);
    Map<Short, Set<TopicReplicationFactorAnomalyEntry>> badTopicsByReplicationFactor = populateBadTopicsByReplicationFactor(topicsToCheck, cluster);
    if (!badTopicsByReplicationFactor.isEmpty()) {
      return Collections.singleton(createTopicReplicationFactorAnomaly(badTopicsByReplicationFactor, _targetReplicationFactor));
    }
  }
  return Collections.emptySet();
}
 

/**
 * Check partitions in the given cluster for existence of offline replicas, and get the first such partition.
 * If there no such partitions are identified, return {@code null}.
 *
 * @param cluster The current cluster state.
 * @return The first identified partition with offline replicas, {@code null} if no such partition exists.
 */
public static PartitionInfo partitionWithOfflineReplicas(Cluster cluster) {
  for (String topic : cluster.topics()) {
    for (PartitionInfo partitionInfo : cluster.partitionsForTopic(topic)) {
      if (partitionInfo.offlineReplicas().length > 0) {
        return partitionInfo;
      }
    }
  }
  return null;
}
 

/**
 * @return True if the metadata has changed, false otherwise.
 */
public static boolean metadataChanged(Cluster prev, Cluster curr) {
  // Broker has changed.
  Set<Node> prevNodeSet = new HashSet<>(prev.nodes());
  if (prevNodeSet.size() != curr.nodes().size()) {
    return true;
  }
  prevNodeSet.removeAll(curr.nodes());
  if (!prevNodeSet.isEmpty()) {
    return true;
  }
  // Topic has changed
  if (!prev.topics().equals(curr.topics())) {
    return true;
  }

  // partition has changed.
  for (String topic : prev.topics()) {
    if (!prev.partitionCountForTopic(topic).equals(curr.partitionCountForTopic(topic))) {
      return true;
    }
    for (PartitionInfo prevPartInfo : prev.partitionsForTopic(topic)) {
      PartitionInfo currPartInfo = curr.partition(new TopicPartition(prevPartInfo.topic(), prevPartInfo.partition()));
      if (leaderChanged(prevPartInfo, currPartInfo) || replicaListChanged(prevPartInfo, currPartInfo)) {
        return true;
      }
    }
  }
  return false;
}
 

/**
 * @param cluster Kafka cluster.
 * @return All the brokers in the cluster that host at least one replica.
 */
static Set<Integer> brokersWithReplicas(Cluster cluster) {
  Set<Integer> allBrokers = new HashSet<>();
  for (String topic : cluster.topics()) {
    for (PartitionInfo partition : cluster.partitionsForTopic(topic)) {
      Arrays.stream(partition.replicas()).map(Node::id).forEach(allBrokers::add);
    }
  }
  return allBrokers;
}
 

@Override
public Set<TopicPartition> assignPartitions(Cluster cluster) {
  // Create an array to host the assignment of the metric fetcher.
  Set<TopicPartition> assignment = new HashSet<>();
  for (String topic : cluster.topics()) {
    List<PartitionInfo> partitionsForTopic = cluster.partitionsForTopic(topic);
    for (PartitionInfo partitionInfo : partitionsForTopic) {
      assignment.add(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()));
    }
  }
  LOG.trace("Partition assignment for metric fetcher: {}", assignment);
  return assignment;
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();
    if (keyBytes == null) {
        int nextValue = roundRobin.getAndIncrement();
        return Utils.toPositive(nextValue) % numPartitions;
    } else {
        // hash the keyBytes to choose a partition
        return Utils.toPositive(xxHasher.hash(keyBytes, 0, keyBytes.length, SEED)) % numPartitions;
    }
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();
    int partitionNum = 0;
    try {
        partitionNum = Utils.murmur2(keyBytes);
    } catch (Exception e) {
        partitionNum = key.hashCode();
    }

    return Math.abs(partitionNum % numPartitions);
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
  List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
  int numPartitions = partitions.size();
  int p = counter % numPartitions;
  counter++;
  return partitions.get(p).partition();
}
 

private static int getPartition(String topic, Cluster cluster, int hash) {
    List<PartitionInfo> partitions = cluster.availablePartitionsForTopic(topic);
    if (partitions.isEmpty()) {
        LOG.warn("No available partitions for {} therefore using total partition count for calculations.", topic);
        partitions = cluster.partitionsForTopic(topic);
    }

    int index = Math.abs(hash) % partitions.size();
    return partitions.get(index).partition();
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();
    if (null == keyBytes) {
        return counter.getAndIncrement() % numPartitions;
    } else
        return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions;
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();

    return messageCount.incrementAndGet() % numPartitions;
}
 

@Override
public int partition(
  String topic,
  Object key,
  byte[] keyBytes,
  Object value,
  byte[] valueBytes,
  Cluster cluster
) {
  List<PartitionInfo> partitionInfos = cluster.partitionsForTopic(topic);
  int availablePartitions = partitionInfos.size();
  return random.nextInt(availablePartitions);
}
 

@Override
public int partition(String topic, Object key, byte[] keyBytes,
                     Object value, byte[] valueBytes, Cluster cluster) {
    List<PartitionInfo> partitions = cluster.partitionsForTopic(topic);
    int numPartitions = partitions.size();
    if (null == keyBytes) {
        return counter.getAndIncrement() % numPartitions;
    } else
        return Utils.toPositive(Utils.murmur2(keyBytes)) % numPartitions;
}