Java Code Examples for org.apache.kafka.clients.consumer.KafkaConsumer#beginningOffsets()

The following examples show how to use org.apache.kafka.clients.consumer.KafkaConsumer#beginningOffsets() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CheckBeginingOffset.java    From BigData-In-Practice with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) {
    KafkaConsumer<String, String> kafkaConsumer = createNewConsumer();
    List<PartitionInfo> partitions = kafkaConsumer.partitionsFor("topic-monitor");
    List<TopicPartition> tpList = partitions.stream()
            .map(pInfo -> new TopicPartition(pInfo.topic(), pInfo.partition()))
            .collect(toList());
    Map<TopicPartition, Long> beginningOffsets =
            kafkaConsumer.beginningOffsets(tpList);
    System.out.println(beginningOffsets);
}
Example 2
Source File: TopicMessageCounter.java    From ja-micro with Apache License 2.0 6 votes vote down vote up 
/**
 * Gets the total message count for the topic.
 * <b>WARNING: Don't use with compacted topics</b>
 */
@SuppressWarnings("unchecked")
public long getCount(String kafkaBrokers, String topic) {
    KafkaConsumer consumer = buildConsumer(kafkaBrokers);
    try {
        @SuppressWarnings("unchecked")
        Map<String, List<PartitionInfo>> topics = consumer.listTopics();
        List<PartitionInfo> partitionInfos = topics.get(topic);
        if (partitionInfos == null) {
            logger.warn("Partition information was not found for topic {}", topic);
            return 0;
        } else {
            Collection<TopicPartition> partitions = new ArrayList<>();
            for (PartitionInfo partitionInfo : partitionInfos) {
                TopicPartition partition = new TopicPartition(topic, partitionInfo.partition());
                partitions.add(partition);
            }
            Map<TopicPartition, Long> endingOffsets = consumer.endOffsets(partitions);
            Map<TopicPartition, Long> beginningOffsets = consumer.beginningOffsets(partitions);
            return diffOffsets(beginningOffsets, endingOffsets);
        }
    } finally {
        consumer.close();
    }
}
Example 3
Source File: KafkaUtils.java    From kafka-spark-consumer with Apache License 2.0 6 votes vote down vote up 
public static long getOffset(
  KafkaConsumer<byte[], byte[]> consumer, String topic, int partition, boolean forceFromStart) {
  TopicPartition  topicAndPartition =
    new TopicPartition (topic, partition);
  Map<TopicPartition, Long > offsetMap = null;
  if(forceFromStart) {
    offsetMap = consumer.beginningOffsets(Arrays.asList(topicAndPartition));
  } else {
    offsetMap = consumer.endOffsets(Arrays.asList(topicAndPartition));
  }
 
  if(offsetMap.get(topicAndPartition) != null ) {
    return offsetMap.get(topicAndPartition);
  } else {
    return NO_OFFSET;
  }
}
Example 4
Source File: CheckBeginingOffset.java    From kafka_book_demo with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) {
KafkaConsumer<String, String> kafkaConsumer = createNewConsumer();
List<PartitionInfo> partitions = kafkaConsumer.partitionsFor("topic-monitor");
List<TopicPartition> tpList = partitions.stream()
        .map(pInfo -> new TopicPartition(pInfo.topic(), pInfo.partition()))
        .collect(toList());
Map<TopicPartition, Long> beginningOffsets =
        kafkaConsumer.beginningOffsets(tpList);
System.out.println(beginningOffsets);
}
Example 5
Source File: KafkaServiceImpl.java    From kafka-eagle with Apache License 2.0 5 votes vote down vote up 
/**
 * Get kafka 0.10.x topic real logsize by partitionid.
 */
public long getKafkaRealLogSize(String clusterAlias, String topic, int partitionid) {
	long realLogSize = 0L;
	Properties props = new Properties();
	props.put(ConsumerConfig.GROUP_ID_CONFIG, Kafka.KAFKA_EAGLE_SYSTEM_GROUP);
	props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias));
	props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) {
		sasl(props, clusterAlias);
	}
	if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) {
		ssl(props, clusterAlias);
	}
	KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
	TopicPartition tp = new TopicPartition(topic, partitionid);
	consumer.assign(Collections.singleton(tp));
	java.util.Map<TopicPartition, Long> endLogSize = consumer.endOffsets(Collections.singleton(tp));
	java.util.Map<TopicPartition, Long> startLogSize = consumer.beginningOffsets(Collections.singleton(tp));
	try {
		realLogSize = endLogSize.get(tp).longValue() - startLogSize.get(tp).longValue();
	} catch (Exception e) {
		LOG.error("Get real topic logsize by partition list has error, msg is " + e.getMessage());
		e.printStackTrace();
	} finally {
		if (consumer != null) {
			consumer.close();
		}
	}
	return realLogSize;
}
Example 6
Source File: ParallelWebKafkaConsumer.java    From kafka-webview with MIT License 5 votes vote down vote up 
private List<PartitionOffset> getHeadOffsets(final KafkaConsumer<?,?> kafkaConsumer) {
    final Map<TopicPartition, Long> results = kafkaConsumer.beginningOffsets(getAllPartitions(kafkaConsumer));

    final List<PartitionOffset> offsets = new ArrayList<>();
    for (final Map.Entry<TopicPartition, Long> entry : results.entrySet()) {
        offsets.add(new PartitionOffset(entry.getKey().partition(), entry.getValue()));
    }
    return offsets;
}
Example 7
Source File: DoctorKafkaActionsServlet.java    From doctorkafka with Apache License 2.0 5 votes vote down vote up 
private List<ConsumerRecord<byte[], byte[]>> retrieveActionReportMessages() {
  DoctorKafkaConfig doctorKafkaConfig = DoctorKafkaMain.doctorKafka.getDoctorKafkaConfig();
  String zkUrl = doctorKafkaConfig.getBrokerstatsZkurl();
  String actionReportTopic = doctorKafkaConfig.getActionReportTopic();
  Properties properties =
      OperatorUtil.createKafkaConsumerProperties(zkUrl, OPERATOR_ACTIONS_CONSUMER_GROUP,
          doctorKafkaConfig.getActionReportProducerSecurityProtocol(),
          doctorKafkaConfig.getActionReportProducerSslConfigs());
  KafkaConsumer<byte[], byte[]> consumer = new KafkaConsumer<>(properties);

  TopicPartition operatorReportTopicPartition = new TopicPartition(actionReportTopic, 0);
  List<TopicPartition> tps = new ArrayList<>();
  tps.add(operatorReportTopicPartition);
  consumer.assign(tps);

  Map<TopicPartition, Long> beginOffsets = consumer.beginningOffsets(tps);
  Map<TopicPartition, Long> endOffsets = consumer.endOffsets(tps);
  for (TopicPartition tp : endOffsets.keySet()) {
    long numMessages = endOffsets.get(tp) - beginOffsets.get(tp);
    LOG.info("{} : offsets [{}, {}], num messages : {}",
        tp, beginOffsets.get(tp), endOffsets.get(tp), numMessages);
    consumer.seek(tp, Math.max(beginOffsets.get(tp), endOffsets.get(tp) - NUM_MESSAGES));
  }

  ConsumerRecords<byte[], byte[]> records = consumer.poll(CONSUMER_POLL_TIMEOUT_MS);
  List<ConsumerRecord<byte[], byte[]>> recordList = new ArrayList<>();

  while (!records.isEmpty()) {
    for (ConsumerRecord<byte[], byte[]> record : records) {
      recordList.add(record);
    }
    records = consumer.poll(CONSUMER_POLL_TIMEOUT_MS);
  }
  LOG.info("Read {} messages", recordList.size());
  return recordList;
}
Example 8
Source File: KafkaOffsetGen.java    From hudi with Apache License 2.0 5 votes vote down vote up 
private Map<TopicPartition, Long> checkupValidOffsets(KafkaConsumer consumer,
                                                      Option<String> lastCheckpointStr, Set<TopicPartition> topicPartitions) {
  Map<TopicPartition, Long> checkpointOffsets = CheckpointUtils.strToOffsets(lastCheckpointStr.get());
  Map<TopicPartition, Long> earliestOffsets = consumer.beginningOffsets(topicPartitions);

  boolean checkpointOffsetReseter = checkpointOffsets.entrySet().stream()
          .anyMatch(offset -> offset.getValue() < earliestOffsets.get(offset.getKey()));
  return checkpointOffsetReseter ? earliestOffsets : checkpointOffsets;
}
Example 9
Source File: DefaultKafkaClusterProxy.java    From kafka-message-tool with MIT License 4 votes vote down vote up 
private List<TopicsOffsetInfo> getTopicOffsetsFor(String consumerGroupId,
                                                  Map<TopicPartition, Object> topicPartitionsCurrentOffset) {
    final Set<TopicPartition> topicPartitions = topicPartitionsCurrentOffset.keySet();
    final List<TopicsOffsetInfo> result = new ArrayList<>();

    final KafkaConsumer<String, String> consumer = createOffsetInfoConsumerFor(consumerGroupId);
    final Map<TopicPartition, Long> beggingOffsets = consumer.beginningOffsets(topicPartitions);
    final Map<TopicPartition, Long> endOffsets = consumer.endOffsets(topicPartitions);

    for (Map.Entry<TopicPartition, Long> entry : beggingOffsets.entrySet()) {

        final TopicPartition topicPartition = entry.getKey();

        if (!endOffsets.containsKey(topicPartition)) {
            continue;
        }

        String currentOffset = NOT_FOUND_STRING;
        String lag = NOT_FOUND_STRING;

        final Optional<Long> optionalOffsetForPartition = getOptionalOffsetForPartition(topicPartitionsCurrentOffset,
                                                                                        topicPartition);

        final String topicName = topicPartition.topic();
        final String partition = String.valueOf(topicPartition.partition());
        final Long startOffsetLong = entry.getValue();
        final String beggingOffset = String.valueOf(startOffsetLong);
        final Long endOffsetLong = endOffsets.get(topicPartition);
        final String endOffset = String.valueOf(endOffsetLong);
        final String msgCount = String.valueOf(endOffsetLong - startOffsetLong);

        if (optionalOffsetForPartition.isPresent()) {
            final Long currentOffsetLong = optionalOffsetForPartition.get();
            currentOffset = String.valueOf(currentOffsetLong);
            lag = String.valueOf(endOffsetLong - currentOffsetLong);
        }

        final TopicsOffsetInfo topicsOffsetInfo = new TopicsOffsetInfo(topicName,
                                                                       beggingOffset,
                                                                       endOffset,
                                                                       consumerGroupId,
                                                                       partition,
                                                                       msgCount,
                                                                       currentOffset,
                                                                       lag);
        result.add(topicsOffsetInfo);
    }

    Logger.debug("Topic offsets: " + result);
    return result;
}