Java Code Examples for org.apache.kafka.clients.consumer.OffsetCommitCallback

The following examples show how to use org.apache.kafka.clients.consumer.OffsetCommitCallback. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
private void commitOffsets(Map<TopicPartition, OffsetAndMetadata> offsets,
                           boolean ignoreConsumerHighWatermark,
                           OffsetCommitCallback callback,
                           boolean sync,
                           Duration timeout) {
  Map<TopicPartition, OffsetAndMetadata> offsetsToCommit =
      _consumerRecordsProcessor.safeOffsetsToCommit(offsets, ignoreConsumerHighWatermark);
  if (sync) {
    if (timeout == null) {
      LOG.trace("Committing offsets synchronously: {}", offsetsToCommit);
      _kafkaConsumer.commitSync(offsetsToCommit);
    } else {
      LOG.trace("Committing offsets synchronously with timeout {} ms: {}", timeout.toMillis(), offsetsToCommit);
      _kafkaConsumer.commitSync(offsetsToCommit, timeout);
    }
  } else {
    LOG.trace("Committing offsets asynchronously: {}", offsetsToCommit);
    _offsetCommitCallback.setUserCallback(callback);
    _kafkaConsumer.commitAsync(offsetsToCommit, _offsetCommitCallback);
  }
}
 
Example 2
Source Project: kbear   Source File: ConsumerProxy.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
    ObjectExtension.requireNonNull(offsets, "offsets");

    runWithoutConcurrency(() -> {
        Map<String, Map<TopicPartition, OffsetAndMetadata>> byTopic = new HashMap<>();
        offsets.forEach((tp, oam) -> byTopic.computeIfAbsent(tp.topic(), k -> new HashMap<>()).put(tp, oam));
        byTopic.forEach((t, os) -> _consumerHolders.get(t).getConsumer().commitAsync(os, callback));
    });
}
 
Example 3
Source Project: smallrye-reactive-messaging   Source File: KafkaUsage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Use the supplied function to asynchronously consume messages from the cluster.
 *
 * @param groupId the name of the group; may not be null
 * @param clientId the name of the client; may not be null
 * @param autoOffsetReset how to pick a starting offset when there is no initial offset in ZooKeeper or if an offset is
 *        out of range; may be null for the default to be used
 * @param keyDeserializer the deserializer for the keys; may not be null
 * @param valueDeserializer the deserializer for the values; may not be null
 * @param continuation the function that determines if the consumer should continue; may not be null
 * @param offsetCommitCallback the callback that should be used after committing offsets; may be null if offsets are
 *        not to be committed
 * @param completion the function to call when the consumer terminates; may be null
 * @param topics the set of topics to consume; may not be null or empty
 * @param consumerFunction the function to consume the messages; may not be null
 */
public <K, V> void consume(String groupId, String clientId, OffsetResetStrategy autoOffsetReset,
        Deserializer<K> keyDeserializer, Deserializer<V> valueDeserializer,
        BooleanSupplier continuation, OffsetCommitCallback offsetCommitCallback, Runnable completion,
        Collection<String> topics,
        java.util.function.Consumer<ConsumerRecord<K, V>> consumerFunction) {
    Properties props = getConsumerProperties(groupId, clientId, autoOffsetReset);
    Thread t = new Thread(() -> {
        LOGGER.infof("Starting consumer %s to read messages", clientId);
        try (KafkaConsumer<K, V> consumer = new KafkaConsumer<>(props, keyDeserializer, valueDeserializer)) {
            consumer.subscribe(new ArrayList<>(topics));
            while (continuation.getAsBoolean()) {
                consumer.poll(Duration.ofMillis(10)).forEach(record -> {
                    LOGGER.infof("Consumer %s: consuming message %s", clientId, record);
                    consumerFunction.accept(record);
                    if (offsetCommitCallback != null) {
                        consumer.commitAsync(offsetCommitCallback);
                    }
                });
            }
        } finally {
            if (completion != null) {
                completion.run();
            }
            LOGGER.debugf("Stopping consumer %s", clientId);
        }
    });
    t.setName(clientId + "-thread");
    t.start();
}
 
Example 4
private OffsetAndMetadata commitAndRetrieveOffsets(
    LiKafkaConsumer<String, String> consumer,
    TopicPartition tp, Map<TopicPartition,
    OffsetAndMetadata> offsetMap) throws Exception {
  final AtomicBoolean callbackFired = new AtomicBoolean(false);
  final AtomicReference<Exception> offsetCommitIssue = new AtomicReference<>(null);
  OffsetAndMetadata committed = null;
  long now = System.currentTimeMillis();
  long deadline = now + TimeUnit.MINUTES.toMillis(1);
  while (System.currentTimeMillis() < deadline) {
    //call commitAsync, wait for a NON-NULL return value (see https://issues.apache.org/jira/browse/KAFKA-6183)
    OffsetCommitCallback commitCallback = new OffsetCommitCallback() {
      @Override
      public void onComplete(Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap, Exception e) {
        if (e != null) {
          offsetCommitIssue.set(e);
        }
        callbackFired.set(true);
      }
    };
    if (offsetMap != null) {
      consumer.commitAsync(offsetMap, commitCallback);
    } else {
      consumer.commitAsync(commitCallback);
    }
    while (!callbackFired.get()) {
      consumer.poll(20);
    }
    Assert.assertNull(offsetCommitIssue.get(), "offset commit failed");
    committed = consumer.committed(tp);
    if (committed != null) {
      break;
    }
    Thread.sleep(100);
  }
  assertNotNull(committed, "unable to retrieve committed offsets within timeout");
  return committed;
}
 
Example 5
@Override
public void commitAsync(OffsetCommitCallback callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    delegate.commitAsync(callback);
  }
}
 
Example 6
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  try (
      @SuppressWarnings("unused") CloseableLock uLock = new CloseableLock(userLock);
      @SuppressWarnings("unused") CloseableLock srLock = new CloseableLock(delegateLock.readLock())
  ) {
    verifyOpen();
    delegate.commitAsync(offsets, callback);
  }
}
 
Example 7
Source Project: jeesuite-libs   Source File: NewApiTopicConsumer.java    License: Apache License 2.0 5 votes vote down vote up
private void commitOffsets(ConsumerWorker worker) {
	
	KafkaConsumer<String, Serializable> consumer = worker.consumer;
	if(worker.isCommiting())return;
	worker.setCommiting(true);
	try {

		if(worker.uncommittedOffsetMap.isEmpty())return ;
		
		logger.debug("committing the offsets : {}", worker.uncommittedOffsetMap);
		consumer.commitAsync(worker.uncommittedOffsetMap, new OffsetCommitCallback() {
			@Override
			public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
				//
				worker.setCommiting(false);
				if(exception == null){
					worker.resetUncommittedOffsetMap();
					logger.debug("committed the offsets : {}",offsets);
				}else{
					logger.error("committ the offsets error",exception);
				}
			}
		});
	} finally {
		
	}
}
 
Example 8
Source Project: kbear   Source File: ConsumerProxy.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(OffsetCommitCallback callback) {
    runWithoutConcurrency(() -> _consumerHolders.values().forEach(c -> c.getConsumer().commitAsync(callback)));
}
 
Example 9
Source Project: kafka-graphs   Source File: PregelConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(OffsetCommitCallback callback) {
    kafkaConsumer.commitAsync(callback);
}
 
Example 10
Source Project: kafka-graphs   Source File: PregelConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
    kafkaConsumer.commitAsync(offsets, callback);
}
 
Example 11
Source Project: java-kafka-client   Source File: TracingKafkaConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(OffsetCommitCallback callback) {
  consumer.commitAsync(callback);
}
 
Example 12
Source Project: java-kafka-client   Source File: TracingKafkaConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets,
    OffsetCommitCallback callback) {
  consumer.commitAsync(offsets, callback);
}
 
Example 13
public void setUserCallback(OffsetCommitCallback userCallback) {
  _userCallback = userCallback;
}
 
Example 14
@Override
public void commitAsync(OffsetCommitCallback callback) {
  // preserve the high watermark.
  commitOffsets(currentOffsetAndMetadataMap(), false, callback, false, null);
}
 
Example 15
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  // Ignore the high watermark.
  commitOffsets(offsets, true, callback, false, null);
}
 
Example 16
@Override
public void commitAsync(OffsetCommitCallback callback) {
  _delegate.commitAsync(callback);
}
 
Example 17
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  _delegate.commitAsync(offsets, callback);
}
 
Example 18
Source Project: kafka-monitor   Source File: NewConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback) {
  _consumer.commitAsync(offsets, callback);
}
 
Example 19
Source Project: kafka-monitor   Source File: NewConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(OffsetCommitCallback callback) {
  _consumer.commitAsync(callback);
}
 
Example 20
Source Project: kafka-monitor   Source File: ConsumeService.java    License: Apache License 2.0 4 votes vote down vote up
private void consume() throws Exception {
  /* Delay 1 second to reduce the chance that consumer creates topic before TopicManagementService */
  Thread.sleep(1000);

  Map<Integer, Long> nextIndexes = new HashMap<>();

  while (_running.get()) {
    BaseConsumerRecord record;
    try {
      record = _baseConsumer.receive();
    } catch (Exception e) {
      _sensors._consumeError.record();
      LOG.warn(_name + "/ConsumeService failed to receive record", e);
      /* Avoid busy while loop */
      //noinspection BusyWait
      Thread.sleep(CONSUME_THREAD_SLEEP_MS);
      continue;
    }

    if (record == null) continue;

    GenericRecord avroRecord = null;
    try {
      avroRecord = Utils.genericRecordFromJson(record.value());
    } catch (Exception exception) {
      LOG.error("An exception occurred while getting avro record.", exception);
    }

    if (avroRecord == null) {
      _sensors._consumeError.record();
      continue;
    }
    int partition = record.partition();
    /* Commit availability and commit latency service */
    /* Call commitAsync, wait for a NON-NULL return value (see https://issues.apache.org/jira/browse/KAFKA-6183) */
    OffsetCommitCallback commitCallback = new OffsetCommitCallback() {
      @Override
      public void onComplete(Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap, Exception kafkaException) {
        if (kafkaException != null) {
          LOG.error("Exception while trying to perform an asynchronous commit.", kafkaException);
          _commitAvailabilityMetrics._failedCommitOffsets.record();
        } else {
          _commitAvailabilityMetrics._offsetsCommitted.record();
          _commitLatencyMetrics.recordCommitComplete();
        }
      }
    };

    /* Current timestamp to perform subtraction*/
    long currTimeMillis = System.currentTimeMillis();

    /* 4 seconds consumer offset commit interval. */
    long timeDiffMillis = TimeUnit.SECONDS.toMillis(COMMIT_TIME_INTERVAL);

    if (currTimeMillis - _baseConsumer.lastCommitted() >= timeDiffMillis) {
      /* commit the consumer offset asynchronously with a callback. */
      _baseConsumer.commitAsync(commitCallback);
      _commitLatencyMetrics.recordCommitStart();
      /* Record the current time for the committed consumer offset */
      _baseConsumer.updateLastCommit();
    }
    /* Finished consumer offset commit service. */

    long index = (Long) avroRecord.get(DefaultTopicSchema.INDEX_FIELD.name());
    long currMs = System.currentTimeMillis();
    long prevMs = (Long) avroRecord.get(DefaultTopicSchema.TIME_FIELD.name());

    _sensors._recordsConsumed.record();
    _sensors._bytesConsumed.record(record.value().length());
    _sensors._recordsDelay.record(currMs - prevMs);

    if (currMs - prevMs > _latencySlaMs)
      _sensors._recordsDelayed.record();

    if (index == -1L || !nextIndexes.containsKey(partition)) {
      nextIndexes.put(partition, -1L);
      continue;
    }

    long nextIndex = nextIndexes.get(partition);

    if (nextIndex == -1 || index == nextIndex) {
      nextIndexes.put(partition, index + 1);

    } else if (index < nextIndex) {
      _sensors._recordsDuplicated.record();
    } else if (index > nextIndex) {
      nextIndexes.put(partition, index + 1);
      long numLostRecords = index - nextIndex;
      _sensors._recordsLost.record(numLostRecords);
      LOG.info("_recordsLost recorded: Avro record current index: {} at timestamp {}. Next index: {}. Lost {} records.", index, currMs, nextIndex, numLostRecords);
    }
  }
  /* end of consume() while loop */
}
 
Example 21
Source Project: kafka-monitor   Source File: ConsumeServiceTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Sample ConsumeService instance for unit testing
 * @return Sample ConsumeService object.
 * @throws Exception should the ConsumeService creation fail or throws an error / exception
 */
private ConsumeService consumeService() throws Exception {
  LOG.info("Creating an instance of Consume Service for testing..");

  ConsumerFactory consumerFactory = Mockito.mock(ConsumerFactory.class);
  AdminClient adminClient = Mockito.mock(AdminClient.class);
  KMBaseConsumer kmBaseConsumer = Mockito.mock(KMBaseConsumer.class);

  Mockito.when(consumerFactory.adminClient()).thenReturn(adminClient);
  Mockito.when(consumerFactory.latencySlaMs()).thenReturn(20000);
  Mockito.when(consumerFactory.baseConsumer()).thenReturn(kmBaseConsumer);
  Mockito.when(consumerFactory.topic()).thenReturn(TOPIC);

  /* LATENCY_PERCENTILE_MAX_MS_CONFIG, */
  Mockito.when(consumerFactory.latencyPercentileMaxMs()).thenReturn(5000);

  /* LATENCY_PERCENTILE_GRANULARITY_MS_CONFIG */
  Mockito.when(consumerFactory.latencyPercentileGranularityMs()).thenReturn(1);

  /* define return value */
  Mockito.when(kmBaseConsumer.lastCommitted()).thenReturn(MOCK_LAST_COMMITTED_OFFSET);
  Mockito.when(kmBaseConsumer.committed(Mockito.any())).thenReturn(new OffsetAndMetadata(FIRST_OFFSET));
  Mockito.doAnswer(new Answer<Void>() {
    @Override
    public Void answer(InvocationOnMock invocationOnMock) {
      OffsetCommitCallback callback = invocationOnMock.getArgument(0);
      Map<TopicPartition, OffsetAndMetadata> committedOffsets = new HashMap<>();
      committedOffsets.put(new TopicPartition(TOPIC, PARTITION), new OffsetAndMetadata(FIRST_OFFSET));
      callback.onComplete(committedOffsets, null);

      return null;
    }
  }).when(kmBaseConsumer).commitAsync(Mockito.any(OffsetCommitCallback.class));


  /* avro record to KmBaseConsumer record */
  Mockito.when(kmBaseConsumer.receive()).thenReturn(
      new BaseConsumerRecord(TOPIC, PARTITION, SECOND_OFFSET, "key",
          Utils.jsonFromFields(TOPIC, 2, 6000, "producerId", 2)));

  CompletableFuture<Void> topicPartitionResult = new CompletableFuture<>();
  topicPartitionResult.complete(null);

  return new ConsumeService(TAG_NAME_VALUE, topicPartitionResult, consumerFactory);
}
 
Example 22
Source Project: attic-apex-malhar   Source File: KafkaConsumer010.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Commit the specified offsets for the specified list of topics and partitions to Kafka.
 * @param offsets given offsets
 * @param callback Callback to invoke when the commit completes
 */
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback)
{
  consumer.commitAsync(offsets, callback);
}
 
Example 23
Source Project: attic-apex-malhar   Source File: KafkaConsumer09.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Commit the specified offsets for the specified list of topics and partitions to Kafka.
 * @param offsets given offsets
 * @param callback Callback to invoke when the commit completes
 */
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback)
{
  consumer.commitAsync(offsets, callback);
}
 
Example 24
Source Project: flink   Source File: KafkaConsumerThreadTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(OffsetCommitCallback offsetCommitCallback) {
}
 
Example 25
Source Project: flink   Source File: KafkaConsumerThreadTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void commitAsync(Map<TopicPartition, OffsetAndMetadata> map, OffsetCommitCallback offsetCommitCallback) {
}
 
Example 26
Source Project: brave   Source File: TracingConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override public void commitAsync(OffsetCommitCallback callback) {
  delegate.commitAsync(callback);
}
 
Example 27
Source Project: brave   Source File: TracingConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override public void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets,
  OffsetCommitCallback callback) {
  delegate.commitAsync(offsets, callback);
}
 
Example 28
/**
 * {@inheritDoc}
 * <p>
 * This method is large message transparent.
 * <p>
 * With large message support, the consumer will internally translate the user provided offsets to the safe
 * offsets for the corresponding partitions (see {@link #safeOffset(TopicPartition, long)}. The actual offset
 * committed to Kafka can be retrieved through method {@link #committedSafeOffset(TopicPartition)}. The arguments
 * passed to the callback is large message transparent.
 */
@Override
@InterfaceOrigin.ApacheKafka
void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback);
 
Example 29
Source Project: attic-apex-malhar   Source File: AbstractKafkaConsumer.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Commit the specified offsets for the specified list of topics and partitions to Kafka.
 * @param offsets given offsets
 * @param callback Callback to invoke when the commit completes
 */
void commitAsync(Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback);
 
Example 30
Source Project: kafka-monitor   Source File: KMBaseConsumer.java    License: Apache License 2.0 votes vote down vote up
void commitAsync(final Map<TopicPartition, OffsetAndMetadata> offsets, OffsetCommitCallback callback);