org.apache.kafka.clients.consumer.InvalidOffsetException Java Examples

The following examples show how to use org.apache.kafka.clients.consumer.InvalidOffsetException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: kcache   Author: rayokota   File: KafkaCache.java    License: Apache License 2.0 6 votes vote down vote up
private void readToEndOffsets() throws IOException {
    Set<TopicPartition> assignment = consumer.assignment();
    Map<TopicPartition, Long> endOffsets = consumer.endOffsets(assignment);
    log.trace("Reading to end of offsets {}", endOffsets);

    int count = 0;
    while (!hasReadToEndOffsets(endOffsets)) {
        try {
            count += poll();
        } catch (InvalidOffsetException e) {
            if (localCache.isPersistent()) {
                localCache.close();
                localCache.destroy();
                localCache.init();
            }
            log.warn("Seeking to beginning due to invalid offset", e);
            consumer.seekToBeginning(assignment);
            count = 0;
        }
    }
    log.info("During init or sync, processed {} records from topic {}", count, topic);
}
 
Example #2
Source Project: li-apache-kafka-clients   Author: linkedin   File: LiKafkaConsumerImpl.java    License: BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * We don't let the underlying open source consumer reset offsets so we need to do that here.
 */
private void handleInvalidOffsetException(InvalidOffsetException oe) throws InvalidOffsetException {
  switch (_offsetResetStrategy) {
    case EARLIEST:
      LOG.warn("Invalid positions of {} due to {}. Resetting position to the earliest.", oe.partitions(),
          oe.getClass().getSimpleName());
      seekToBeginning(oe.partitions());
      break;
    case LATEST:
      LOG.warn("Invalid positions of {} due to {}. Resetting position to the latest.", oe.partitions(), oe.getClass().getSimpleName());
      seekToEnd(oe.partitions());
      break;
    case LICLOSEST:
      LOG.warn("Invalid positions of {} due to {}. Resetting position to the li_closest.", oe.partitions(),
          oe.getClass().getSimpleName());
      handleLiClosestResetStrategy(oe);
      break;
    default:
      throw oe;
  }
}
 
Example #3
Source Project: kafka-spark-consumer   Author: dibbhatt   File: KafkaUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static ConsumerRecords<byte[], byte[]> fetchMessages(
  KafkaConfig config, KafkaConsumer<byte[], byte[]> consumer, Partition partition,
  long offset) {
  String topic = (String) config._stateConf.get(Config.KAFKA_TOPIC);
  int partitionId = partition.partition;
  TopicPartition  topicAndPartition = new TopicPartition (topic, partitionId);
  consumer.seek(topicAndPartition, offset);
  ConsumerRecords<byte[], byte[]> records;
  try {
    records = consumer.poll(config._fillFreqMs / 2);
  } catch(InvalidOffsetException ex) {
    throw new OutOfRangeException(ex.getMessage());
  } catch (Exception e) {
    if (e instanceof KafkaException || e instanceof ConnectException
      || e instanceof SocketTimeoutException || e instanceof IOException
      || e instanceof UnresolvedAddressException) {

      LOG.warn("Network error when fetching messages:", e);
      throw new FailedFetchException(e);
    } else {
      throw new RuntimeException(e);
    }
  }
  return records;
}
 
Example #4
Source Project: li-apache-kafka-clients   Author: linkedin   File: LiKafkaConsumerImpl.java    License: BSD 2-Clause "Simplified" License 4 votes vote down vote up
/**
 * This method handles the OffsetResetStrategy="LICLOSEST" offset reset strategy.
 *
 * The semantics of this strategy is defined as follows:
 * Consumer will {@link #seekToBeginning(Collection)} when InvalidOffsetException occurs due to:
 * 1. New Consumer / Expired Commit Offset
 * 2. Fall-off Start (fetch offset < LSO)
 *
 * Consumer will {@link #seekToEnd(Collection)} when InvalidOffsetException occurs due to:
 * 3a. Fall-off End (fetch offset > LEO): Consumer will seek to the end
 * 3b. Fall-off End (fetch offset <= LEO): Consumer will seek to the fetched offset
 *
 * Note: Offset to which we reset may not necessarily be a safe offset. This method invokes 2 blocking calls and does
 * ignore large-message tracking metadata. If we are unable to calculate the bounds, it will throw an
 * IllegalStateException.
 *
 * Design details can be found here - https://docs.google.com/document/d/1zKGXxZiyiRkLJ_d0FCoGALfAo0N7k3hh9NFYhJrbPsw/edit#
 * @param oe InvalidOffsetException
 */
private void handleLiClosestResetStrategy(InvalidOffsetException oe) {
  if (oe instanceof NoOffsetForPartitionException) {  // Case 1
    LOG.info("No valid offsets found. Rewinding to the earliest");
    seekToBeginning(oe.partitions());
  } else if (oe instanceof OffsetOutOfRangeException) {
    Map<TopicPartition, Long> seekBeginningPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekEndPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekFetchedOffsetPartitions = new HashMap<>();
    Set<TopicPartition> boundsUnknownPartitions = new HashSet<>();

    Map<TopicPartition, Long> beginningOffsets = beginningOffsets(oe.partitions());
    Map<TopicPartition, Long> endOffsets = endOffsets(oe.partitions());

    ((OffsetOutOfRangeException) oe).offsetOutOfRangePartitions().forEach((tp, fetchedOffset) -> {
      long beginningOffset = beginningOffsets.getOrDefault(tp, -1L);
      long endOffset = endOffsets.getOrDefault(tp, -1L);
      if (beginningOffset != -1L && endOffset != -1L) {
        if (beginningOffset > fetchedOffset) {  // Case 2
          seekBeginningPartitions.put(tp, beginningOffset);
          return;
        }
        if (endOffset < fetchedOffset) {  // Case 3a
          LOG.debug("Closest offset computed for topic partition {} is the log end offset {}. ", tp, fetchedOffset);
          seekEndPartitions.put(tp, endOffset);
        } else {  // Case 3b: endOffset >= fetchedOffset
          LOG.debug("Closest offset computed for topic partition {} is the fetched offset {}. ", tp, fetchedOffset);
          seekFetchedOffsetPartitions.put(tp, fetchedOffset);
        }
      } else {
        // can't handle reset if the either bound values are not known
        // ideally, this should never happen since the listoffsets protocol always returns all requested offset or none
        boundsUnknownPartitions.add(tp);
      }
    });

    if (!boundsUnknownPartitions.isEmpty()) {
      throw new IllegalStateException("Couldn't figure out the closest offset for these topic partitions " +
          boundsUnknownPartitions + "Aborting..");
    }

    if (!seekBeginningPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the beginning offsets returned", seekBeginningPartitions);
      seekBeginningPartitions.forEach(this::seekAndClear);
    }
    if (!seekEndPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the end offsets returned", seekEndPartitions);
      seekEndPartitions.forEach(this::seekAndClear);
    }
    if (!seekFetchedOffsetPartitions.isEmpty()) {
      LOG.info("Seeking to fetched offsets for topic partitions {}. This may indicate a potential loss of data.",
          seekFetchedOffsetPartitions.keySet());
      seekFetchedOffsetPartitions.forEach(this::seekAndClear);
    }
  } else {
    throw oe;
  }
}