Java Code Examples for org.apache.kafka.clients.consumer.OffsetOutOfRangeException

The following examples show how to use org.apache.kafka.clients.consumer.OffsetOutOfRangeException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
/**
 * Calling nextRecord calls fillBuffer() which has special handling for OutOfRangeExceptions.  There is some
 * recursion in this method, but it should not go on forever and yield a StackOverflow exception.  This test verifies
 * that when the KafkaConsumer is relentlessly throwing OutOfRangeExceptions that we stop trying to fill the buffer
 * after five attempts and do not have any other errors.
 */
@Test
public void testNextRecordWithRecursiveOutOfRangeException() {
    // Create mock KafkaConsumer instance
    @SuppressWarnings("unchecked")
    final KafkaConsumer<byte[], byte[]> mockKafkaConsumer = (KafkaConsumer<byte[], byte[]>) mock(KafkaConsumer.class);

    Mockito.when(mockKafkaConsumer.assignment()).thenReturn(
        Stream.of(new TopicPartition("Foobar", 0)).collect(Collectors.toSet())
    );
    Mockito.when(mockKafkaConsumer.partitionsFor(topicName)).thenReturn(Arrays.asList(
        new PartitionInfo(topicName, 0, null, null, null)
    ));
    Mockito.when(mockKafkaConsumer.poll(300)).thenThrow(
        new OffsetOutOfRangeException(new HashMap<>())
    );

    final PersistenceAdapter persistenceAdapter = new InMemoryPersistenceAdapter();
    persistenceAdapter.open(new HashMap<>());

    // Create our consumer
    final Consumer consumer = new Consumer(mockKafkaConsumer);
    consumer.open(
        getDefaultConfig(topicName),
        getDefaultVSpoutId(),
        getDefaultConsumerCohortDefinition(),
        persistenceAdapter,
        new LogRecorder(),
        null
    );

    final Record record = consumer.nextRecord();

    assertNull(record);

    Mockito.verify(mockKafkaConsumer, Mockito.times(5)).poll(300);

    consumer.close();
}
 
Example 2
private long positionMain(TopicPartition partition, Duration timeout) {
  // Not handling large message here. The position will be actual position.
  while (true) { // In kafka 0.10.x we can get an unbounded number of invalid offset exception
    try {
      if (timeout == null) {
        return _kafkaConsumer.position(partition);
      } else {
        return _kafkaConsumer.position(partition, timeout);
      }
    } catch (OffsetOutOfRangeException | NoOffsetForPartitionException oe) {
      handleInvalidOffsetException(oe);
    }
  }
}
 
Example 3
/**
 * Internal method used to fill internal message buffer from kafka.
 *
 * Limited by the number of trips made. This should only be called from {@link #fillBuffer()}.
 */
private void fillBuffer(final int trips) {
    // If our buffer is null, or our iterator is at the end
    if (buffer == null || !bufferIterator.hasNext()) {

        // If we have no assigned partitions to consume from, then don't call poll()
        // The underlying consumer call here does NOT make an API call, so this is safe to call within this loop.
        if (getKafkaConsumer().assignment().isEmpty()) {
            // No assigned partitions, nothing to consume :)
            return;
        }

        // Time to refill the buffer
        try {
            buffer = getKafkaConsumer().poll(300);
        } catch (OffsetOutOfRangeException outOfRangeException) {
            // Handle it
            handleOffsetOutOfRange(outOfRangeException);

            // Clear out so we can attempt next time.
            buffer = null;
            bufferIterator = null;

            // Why 5? Because it's less than 6.
            if (trips >= 5) {
                logger.error(
                    "Attempted to fill the buffer after an OffsetOutOfRangeException, but this was my fifth attempt so I'm bailing."
                );
                // nextRecord() will get called by the VirtualSpout instance soon so we're not giving up, just avoiding a StackOverflow
                // exception on this current run of checks.
                return;
            }

            fillBuffer(trips + 1);
            return;
        }

        // Create new iterator
        bufferIterator = buffer.iterator();
    }
}
 
Example 4
/**
 * This method handles when a partition seek/retrieve request was out of bounds.
 * This happens in two scenarios:
 *  1 - The offset is too old and was cleaned up / removed by the broker.
 *  2 - The offset just plain does not exist.
 *
 * This is particularly nasty in that if the poll() was able to pull SOME messages from
 * SOME partitions before the exception was thrown, those messages are considered "consumed"
 * by KafkaClient, and there's no way to get them w/o seeking back to them for those partitions.
 *
 * This means when we roll back, we may replay some messages :/
 *
 * @param outOfRangeException The exception that was raised by the consumer.
 */
private void handleOffsetOutOfRange(final OffsetOutOfRangeException outOfRangeException) {
    final Set<TopicPartition> resetPartitions = new HashSet<>();

    // Loop over all the partitions in this exception
    for (final TopicPartition topicPartition : outOfRangeException.offsetOutOfRangePartitions().keySet()) {
        // The offset that was in the error
        final long exceptionOffset = outOfRangeException.offsetOutOfRangePartitions().get(topicPartition);
        // What kafka says the last offset is
        final long endingOffset = getKafkaConsumer().endOffsets(Collections.singletonList(topicPartition))
            .get(topicPartition);

        logger.warn("Offset Out of Range for partition {} at offset {}, kafka says last offset in partition is {}",
            topicPartition.partition(), exceptionOffset, endingOffset);

        // We have a hypothesis that the consumer can actually seek past the last message of the topic,
        // this yields this error and we want to catch it and try to back it up just a bit to a place that
        // we can work from.
        if (exceptionOffset >= endingOffset) {
            logger.warn(
                "OutOfRangeException yielded offset {}, which is past our ending offset of {} for {}",
                exceptionOffset,
                endingOffset,
                topicPartition
            );

            // Seek to the end we found above.  The end may have moved since we last asked, which is why we are not doing seekToEnd()
            getKafkaConsumer().seek(
                topicPartition,
                endingOffset
            );

            partitionOffsetsManager.replaceEntry(
                new ConsumerPartition(topicPartition.topic(), topicPartition.partition()),
                endingOffset
            );
        } else {
            resetPartitions.add(topicPartition);
        }
    }

    // All of the error'd partitions we need to seek to earliest available position.
    resetPartitionsToEarliest(resetPartitions);
}
 
Example 5
@Test(dataProvider = "offsetResetStrategies")
public void testOffsetOutOfRangeForStrategy(LiOffsetResetStrategy strategy) throws Exception {
  createTopic(TOPIC1);
  createTopic(TOPIC2);
  produceRecordsWithKafkaProducer();
  Properties props = new Properties();
  props.setProperty("auto.offset.reset", strategy.name());
  props.setProperty("group.id", "testOffsetOutOfRange");

  TopicPartition tp = new TopicPartition(TOPIC1, 0);
  try (LiKafkaConsumer<String, String> consumer = createConsumer(props)) {
    consumer.assign(Collections.singleton(tp));
    ConsumerRecords<String, String> consumerRecords = ConsumerRecords.empty();
    consumer.seek(tp, 0);
    while (consumerRecords.isEmpty()) {
      consumerRecords = consumer.poll(1000);
    }
    consumer.seek(tp, 100000L);
    assertEquals(consumer.position(tp), 100000L);
    switch (strategy) {
      case EARLIEST:
        long expectedEarliestOffset = consumerRecords.iterator().next().offset();
        consumerRecords = ConsumerRecords.empty();
        while (consumerRecords.isEmpty()) {
          consumerRecords = consumer.poll(1000);
        }
        assertEquals(consumerRecords.iterator().next().offset(), expectedEarliestOffset,
            "The offset should have been reset to the earliest offset");
        break;
      case LATEST:
        consumer.poll(1000);
        long expectedLatestOffset = consumer.endOffsets(Collections.singleton(tp)).get(tp);
        assertEquals(consumer.position(tp), expectedLatestOffset,
            "The offset should have been reset to the latest offset");
        break;
      case LICLOSEST:
        long expectedEndOffset = consumer.endOffsets(Collections.singleton(tp)).get(tp);
        consumer.poll(1000);
        long currentPosition = consumer.position(tp);
        assertTrue(currentPosition == expectedEndOffset);
        break;
      case NONE:
        try {
          consumer.poll(1000);
          fail("OffsetOutOfRangeException should have been thrown.");
        } catch (OffsetOutOfRangeException oore) {
          // Expected
        }
        break;
      default:
        fail("Unknown reset strategy " + strategy);
        break;
    }
  }
}
 
Example 6
private ConsumerRecords<K, V> poll(long timeout, boolean includeMetadataInTimeout) {
  ConsumerRecords<K, V> processedRecords;
  // We will keep polling until timeout.
  long now = System.currentTimeMillis();
  long deadline = now + timeout;
  do {
    ConsumerRecordsProcessingException crpe;

    // throw exception to user if the current active (un-paused) topic-partitions has exceptions
    Set<TopicPartition> unPausedTopicPartitions = new HashSet<>(_kafkaConsumer.assignment());
    unPausedTopicPartitions.removeAll(_kafkaConsumer.paused());
    crpe = handleRecordProcessingException(unPausedTopicPartitions);
    if (crpe != null) {
      throw crpe;
    }

    if (_autoCommitEnabled && now > _lastAutoCommitMs + _autoCommitInterval) {
      commitAsync();
      _lastAutoCommitMs = now;
    }
    ConsumerRecords<byte[], byte[]> rawRecords = ConsumerRecords.empty();
    try {
      if (includeMetadataInTimeout) {
        rawRecords = _kafkaConsumer.poll(Duration.ofMillis(deadline - now));
      } else {
        rawRecords = _kafkaConsumer.poll(deadline - now);
      }
    } catch (OffsetOutOfRangeException | NoOffsetForPartitionException oe) {
      handleInvalidOffsetException(oe);
    }

    _lastProcessedResult = _consumerRecordsProcessor.process(rawRecords);
    processedRecords = _lastProcessedResult.consumerRecords();
    // Clear the internal reference.
    _lastProcessedResult.clearRecords();
    // Rewind offset if there are processing exceptions.
    seekToCurrentOffsetsOnRecordProcessingExceptions();

    // this is an optimization
    // if no records were processed try to throw exception in current poll()
    if (processedRecords.isEmpty()) {
      crpe = handleRecordProcessingException(null);
      if (crpe != null) {
        throw crpe;
      }
    }
    now = System.currentTimeMillis();
  } while (processedRecords.isEmpty() && now < deadline);
  return processedRecords;
}
 
Example 7
/**
 * This method handles the OffsetResetStrategy="LICLOSEST" offset reset strategy.
 *
 * The semantics of this strategy is defined as follows:
 * Consumer will {@link #seekToBeginning(Collection)} when InvalidOffsetException occurs due to:
 * 1. New Consumer / Expired Commit Offset
 * 2. Fall-off Start (fetch offset < LSO)
 *
 * Consumer will {@link #seekToEnd(Collection)} when InvalidOffsetException occurs due to:
 * 3a. Fall-off End (fetch offset > LEO): Consumer will seek to the end
 * 3b. Fall-off End (fetch offset <= LEO): Consumer will seek to the fetched offset
 *
 * Note: Offset to which we reset may not necessarily be a safe offset. This method invokes 2 blocking calls and does
 * ignore large-message tracking metadata. If we are unable to calculate the bounds, it will throw an
 * IllegalStateException.
 *
 * Design details can be found here - https://docs.google.com/document/d/1zKGXxZiyiRkLJ_d0FCoGALfAo0N7k3hh9NFYhJrbPsw/edit#
 * @param oe InvalidOffsetException
 */
private void handleLiClosestResetStrategy(InvalidOffsetException oe) {
  if (oe instanceof NoOffsetForPartitionException) {  // Case 1
    LOG.info("No valid offsets found. Rewinding to the earliest");
    seekToBeginning(oe.partitions());
  } else if (oe instanceof OffsetOutOfRangeException) {
    Map<TopicPartition, Long> seekBeginningPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekEndPartitions = new HashMap<>();
    Map<TopicPartition, Long> seekFetchedOffsetPartitions = new HashMap<>();
    Set<TopicPartition> boundsUnknownPartitions = new HashSet<>();

    Map<TopicPartition, Long> beginningOffsets = beginningOffsets(oe.partitions());
    Map<TopicPartition, Long> endOffsets = endOffsets(oe.partitions());

    ((OffsetOutOfRangeException) oe).offsetOutOfRangePartitions().forEach((tp, fetchedOffset) -> {
      long beginningOffset = beginningOffsets.getOrDefault(tp, -1L);
      long endOffset = endOffsets.getOrDefault(tp, -1L);
      if (beginningOffset != -1L && endOffset != -1L) {
        if (beginningOffset > fetchedOffset) {  // Case 2
          seekBeginningPartitions.put(tp, beginningOffset);
          return;
        }
        if (endOffset < fetchedOffset) {  // Case 3a
          LOG.debug("Closest offset computed for topic partition {} is the log end offset {}. ", tp, fetchedOffset);
          seekEndPartitions.put(tp, endOffset);
        } else {  // Case 3b: endOffset >= fetchedOffset
          LOG.debug("Closest offset computed for topic partition {} is the fetched offset {}. ", tp, fetchedOffset);
          seekFetchedOffsetPartitions.put(tp, fetchedOffset);
        }
      } else {
        // can't handle reset if the either bound values are not known
        // ideally, this should never happen since the listoffsets protocol always returns all requested offset or none
        boundsUnknownPartitions.add(tp);
      }
    });

    if (!boundsUnknownPartitions.isEmpty()) {
      throw new IllegalStateException("Couldn't figure out the closest offset for these topic partitions " +
          boundsUnknownPartitions + "Aborting..");
    }

    if (!seekBeginningPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the beginning offsets returned", seekBeginningPartitions);
      seekBeginningPartitions.forEach(this::seekAndClear);
    }
    if (!seekEndPartitions.isEmpty()) {
      LOG.info("Offsets are out of range for partitions {}. Seeking to the end offsets returned", seekEndPartitions);
      seekEndPartitions.forEach(this::seekAndClear);
    }
    if (!seekFetchedOffsetPartitions.isEmpty()) {
      LOG.info("Seeking to fetched offsets for topic partitions {}. This may indicate a potential loss of data.",
          seekFetchedOffsetPartitions.keySet());
      seekFetchedOffsetPartitions.forEach(this::seekAndClear);
    }
  } else {
    throw oe;
  }
}
 
Example 8
/**
 * Handle when Kafka consumer throws OffsetOutOfRangeException. The base behavior is no-op.
 * @param e the Exception
 */
protected void handleOffsetOutOfRangeException(OffsetOutOfRangeException e) {
}