Java Code Examples for org.apache.kafka.clients.consumer.ConsumerRecord

@ReliableOnConsumed(svc = "cat-settle",nextTopic = "CAT_SETTLE_CREATED", nextRetryMax = 2, nextSvcs = {"cat-statement"})
@KafkaListener(topics = "CAT_PAID_TCC_TRY")
public CatStatement onCatPaid_TCC_TRY(ConsumerRecord<String, String> record) {

    ReliableDto dto = dtoConverter.convertOnConsumed(record);
    //--------------

    CatSettle catSettle = new CatSettle();
    catSettle.setId("CAT_SETTLE_TEST");
    catSettle.setName("TRY");

    settleController.create(catSettle);

    CatStatement catStatement = new CatStatement();
    catStatement.setId("TEST_STATEMENT");
    catStatement.setTest("CAT_SETTLE_CREATED");

    return catStatement;

}
 

@Override
public List<ConsumerRecord<K, V>> call() throws Exception {
    try {
        Map<TopicPartition, OffsetAndMetadata> commitBuffer = Maps.newHashMap();
        List<ConsumerRecord<K, V>> polledMessages = new ArrayList<>(numRecordsToPoll);
        while ((polledMessages.size() < numRecordsToPoll) && (!Thread.currentThread().isInterrupted())) {
            ConsumerRecords<K, V> records = consumer.poll(0);
            for (ConsumerRecord<K, V> rec : records) {
                polledMessages.add(rec);
                commitBuffer.put(
                        new TopicPartition(rec.topic(), rec.partition()),
                        new OffsetAndMetadata(rec.offset() + 1)
                );

                if (polledMessages.size() == numRecordsToPoll) {
                    consumer.commitSync(commitBuffer);
                    break;
                }
            }
        }
        return polledMessages;
    } finally {
        consumer.close();
    }
}
 

private void processRecords(final ConsumerRecords<byte[], byte[]> records) {
    records.partitions().stream().forEach(partition -> {
        List<ConsumerRecord<byte[], byte[]>> messages = records.records(partition);
        if (!messages.isEmpty()) {
            //update maximum offset map for this topic partition
            long maxOffset = messages.stream()
                    .mapToLong(record -> record.offset())
                    .max()
                    .getAsLong();

            //write records to content repository and session
            if (demarcatorBytes != null) {
                writeDemarcatedData(getProcessSession(), messages, partition);
            } else if (readerFactory != null && writerFactory != null) {
                writeRecordData(getProcessSession(), messages, partition);
            } else {
                messages.stream().forEach(message -> {
                    writeData(getProcessSession(), message, partition);
                });
            }

            totalMessages += messages.size();
            uncommittedOffsetsMap.put(partition, new OffsetAndMetadata(maxOffset + 1L));
        }
    });
}
 

public static void main(String[] args) {

    Map<String, String> config = new HashMap<>();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    config.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
    config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
    config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

    KafkaConsumer consumer = new KafkaConsumer(config);
    consumer.subscribe(Collections.singleton("my-topic"));

    while (true) {
      ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
      for (ConsumerRecord<String, String> record: records) {
        System.out.println(record);
      }
    }
  }
 

public static void main(String[] args) {
    Properties props = initConfig();
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));

    try {
        while (running.get()) {
            ConsumerRecords<String, String> records = consumer.poll(1000);
            for (ConsumerRecord<String, String> record : records) {
                //do some logical processing.
            }
            consumer.commitSync();
        }
    } finally {
        consumer.close();
    }
}
 

public void start() {
    Collection<String> topics = Collections.singletonList(TOPIC_SERVICE_REQUESTS);
    this.consumer.subscribe(topics);
    LOG.info("Waiting for requests {} ...", serviceId);
    this.running = true;
    while (running) {
        ConsumerRecords<String, Bytes> records = consumer.poll(Duration.ofMillis(10));
        if (!records.isEmpty()) {
            for (ConsumerRecord<String, Bytes> record: records) {
                try {
                    ServiceRequest request = dataMapper.deserialize(record.value(), ServiceRequest.class);
                    LOG.info("Received Request: {}:{}:{}", record.key(), request.getClientId(), request.getTaskId());
                    ServiceResponse response =
                            new ServiceResponse(request.getTaskId(), request.getClientId(), request.getData(), "response:" + request.getData());
                    Bytes bytes = dataMapper.serialize(response);
                    ProducerRecord<String, Bytes> recordReply = new ProducerRecord<>(TOPIC_SERVICE_RESPONSES, response.getTaskId(), bytes);
                    producer.send(recordReply);
                    LOG.info("Response has been send !");
                } catch (IOException e) {
                    LOG.error("Exception: ", e);
                }
            }
        }
    }
    LOG.info("done {}.", serviceId);
}
 

@Override
public void emitTuples()
{
  int count = consumerWrapper.messageSize();
  if (maxTuplesPerWindow > 0) {
    count = Math.min(count, maxTuplesPerWindow - emitCount);
  }
  for (int i = 0; i < count; i++) {
    Pair<String, ConsumerRecord<byte[], byte[]>> tuple = consumerWrapper.pollMessage();
    ConsumerRecord<byte[], byte[]> msg = tuple.getRight();
    emitTuple(tuple.getLeft(), msg);
    AbstractKafkaPartitioner.PartitionMeta pm = new AbstractKafkaPartitioner.PartitionMeta(tuple.getLeft(),
        msg.topic(), msg.partition());
    offsetTrack.put(pm, msg.offset() + 1);
    if (isIdempotent() && !windowStartOffset.containsKey(pm)) {
      windowStartOffset.put(pm, msg.offset());
    }
  }
  emitCount += count;
  processConsumerError();
}
 

static ConsumerRecordsDesc create(Iterator consumerRecordIterator) {
    Set<String> topicSet = new HashSet<String>(1);
    String remoteAddress = null;
    int count = 0;

    while (consumerRecordIterator.hasNext()) {
        Object consumerRecord = consumerRecordIterator.next();
        if (consumerRecord instanceof ConsumerRecord) {
            if (remoteAddress == null) {
                remoteAddress = getRemoteAddress(consumerRecord);
            }

            String topic = ((ConsumerRecord) consumerRecord).topic();
            topicSet.add(topic);
            count++;
        }
    }

    if (count > 0) {
        return new ConsumerRecordsDesc(topicSet, remoteAddress, count);
    }

    return null;
}
 

@Override
public void run() {
    try {
        while (true) {
            ConsumerRecords<String, String> records = kafkaConsumer.poll(Duration.ofMillis(100));
            for (ConsumerRecord<String, String> record : records) {
                //process record.
                System.out.println(getName() + " -> " + record.value());
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        kafkaConsumer.close();
    }
}
 

@Test
public void shouldDecodeBrokenCompoundKeysAsMessageKey() {
    final KafkaDecoder decoder = new KafkaDecoder();
    final ConsumerRecord<String,String> record = new ConsumerRecord<>(
            "ch01",
            0,
            42L,
            1234L, TimestampType.CREATE_TIME,
            -1L, -1, -1,
            "record-key",
            null,
            new RecordHeaders(asList(
                    new RecordHeader("_synapse_msg_partitionKey", "1234".getBytes(UTF_8)),
                    new RecordHeader("_synapse_msg_compactionKey", "key-1234".getBytes(UTF_8))
            ))
    );

    // when
    final TextMessage decodedMessage = decoder.apply(record);

    // then
    assertThat(decodedMessage.getKey().isCompoundKey(), is(false));
    assertThat(decodedMessage.getKey().compactionKey(), is("record-key"));
}
 

@Override
public String deserialize(ConsumerRecord<byte[], byte[]> record) {
	final long offset = record.offset();
	if (offset != nextExpected) {
		numJumps++;
		nextExpected = offset;
		LOG.info("Registered now jump at offset {}", offset);
	}
	nextExpected++;
	try {
		Thread.sleep(10); // slow down data consumption to trigger log eviction
	} catch (InterruptedException e) {
		throw new RuntimeException("Stopping it");
	}
	return "";
}
 

@Test
public void testDeserializeWithMetadata() throws Exception {
	ObjectMapper mapper = new ObjectMapper();
	ObjectNode initialKey = mapper.createObjectNode();
	initialKey.put("index", 4);
	byte[] serializedKey = mapper.writeValueAsBytes(initialKey);

	ObjectNode initialValue = mapper.createObjectNode();
	initialValue.put("word", "world");
	byte[] serializedValue = mapper.writeValueAsBytes(initialValue);

	JSONKeyValueDeserializationSchema schema = new JSONKeyValueDeserializationSchema(true);
	final ConsumerRecord<byte[], byte[]> consumerRecord =
			newConsumerRecord("topic#1", 3, 4L, serializedKey, serializedValue);
	ObjectNode deserializedValue = schema.deserialize(consumerRecord);

	Assert.assertEquals(4, deserializedValue.get("key").get("index").asInt());
	Assert.assertEquals("world", deserializedValue.get("value").get("word").asText());
	Assert.assertEquals("topic#1", deserializedValue.get("metadata").get("topic").asText());
	Assert.assertEquals(4, deserializedValue.get("metadata").get("offset").asInt());
	Assert.assertEquals(3, deserializedValue.get("metadata").get("partition").asInt());
}
 

public void run() {
	Properties props = new Properties();
	props.put("bootstrap.servers", "192.168.99.100:9092");
	props.put("group.id", "geolocationConsumer");
	props.put("key.deserializer", StringDeserializer.class.getName());
	props.put("value.deserializer", StringDeserializer.class.getName());

	try (KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props)) {
		consumer.subscribe(Arrays.asList("geolocations"));
		while (true) {
			ConsumerRecords<String, String> records = consumer.poll(100);
			for (ConsumerRecord<String, String> record : records) {
				System.out.printf("offset = %d, key = %s, value = %s%n", 
						record.offset(), 
						record.key(), 
						record.value());

				REPO.addGeoLocation(GSON.fromJson(record.value(), GeoLocation.class));
			}
		}
	} catch (Exception e) {
		System.err.println("Error while consuming geolocations. Details: " + e.getMessage());
	}
}
 

public void onEventMessage(ConsumerRecord<String, Serializable> record, PublishedEventWrapper eventWrapper) {
    try {
        String topic = record.topic();
        if (topic.equals("operation-events")) {
            log.warn("Topic must not be operation-events");
            return;
        }
        String opId = eventWrapper.getContext().getOpId();
        log.info("opId:" + opId + " EventMessage -> Topic: " + topic
                + " - Sender: " + eventWrapper.getSender() + " - aggregateId: " + eventWrapper.getContext().getCommandContext());

        topicsMap.submitToKey(topic, new EndOffsetSetter(record.partition(), record.offset() + 1));

    } catch (Exception e) {
        log.error("Error While Handling Event:" + e.getMessage(), e);
    }
}
 

@Test
public void shouldInterceptMessage() {
    // given
    final KafkaRecordsConsumer consumer = someKafkaRecordsConsumer(fromHorizon());

    final ConsumerRecord<String, String> record = someRecord(0, 42L, Clock.systemDefaultZone());

    // when
    registry.register(allChannelsWith((m) -> {
        return TextMessage.of(m.getKey(), m.getHeader(), "intercepted");
    }));

    final ConsumerRecords<String,String> records = new ConsumerRecords<>(ImmutableMap.of(
            new TopicPartition("foo", 0),
            singletonList(record))
    );
    consumer.apply(records);

    // then
    verify(dispatcher).accept(of(Key.of("key"), of(fromPosition("0", "42")), "intercepted"));
}
 

@Override
public String deserialize(ConsumerRecord<byte[], byte[]> record) {
	final long offset = record.offset();
	if (offset != nextExpected) {
		numJumps++;
		nextExpected = offset;
		LOG.info("Registered now jump at offset {}", offset);
	}
	nextExpected++;
	try {
		Thread.sleep(10); // slow down data consumption to trigger log eviction
	} catch (InterruptedException e) {
		throw new RuntimeException("Stopping it");
	}
	return "";
}
 

/**
 * Polls a {@link Consumer} until it has either polled too many times without hitting the target number
 * of results, or it hits the target number of results.
 *
 * @param pollMs - How long each poll could take.
 * @param pollIterations - The maximum number of polls that will be attempted.
 * @param targetSize - The number of results to read before stopping.
 * @param consumer - The consumer that will be polled.
 * @return The results that were read from the consumer.
 * @throws Exception If the poll failed.
 */
public static <K, V> List<V> pollForResults(
        final int pollMs,
        final int pollIterations,
        final int targetSize,
        final Consumer<K, V> consumer) throws Exception {
    requireNonNull(consumer);

    final List<V> values = new ArrayList<>();

    int i = 0;
    while(values.size() < targetSize && i < pollIterations) {
        for(final ConsumerRecord<K, V> record : consumer.poll(pollMs)) {
            values.add( record.value() );
        }
        i++;
    }

    return values;
}
 

@Test
public void shouldUseTimestampExtractedFromDDLStatement() throws Exception {
  final String outputStream = "DDL_TIMESTAMP";
  ksqlContext.sql("CREATE STREAM "+  outputStream
      + " WITH(timestamp='ordertime')"
      + " AS SELECT ORDERID, ORDERTIME FROM "
      + avroTimestampStreamName
      + " WHERE ITEMID='ITEM_4';");

  final List<ConsumerRecord> records = testHarness.consumerRecords(outputStream,
      1,
      IntegrationTestHarness.RESULTS_POLL_MAX_TIME_MS);

  final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
  final long timestamp = records.get(0).timestamp();
  assertThat(timestamp, equalTo(dateFormat.parse("2018-01-04").getTime()));
}
 

@Test
public void createTraceTest1() {
    List<ConsumerRecord> consumerRecordList = new ArrayList<ConsumerRecord>();
    consumerRecordList.add(new ConsumerRecord("Test", 1, 1, "hello", "hello too"));

    doReturn(trace).when(traceContext).newTraceObject();
    doReturn(true).when(trace).canSampled();
    doReturn(recorder).when(trace).getSpanRecorder();
    doReturn(consumerRecordList.iterator()).when(consumerRecords).iterator();

    ConsumerMultiRecordEntryPointInterceptor interceptor = new ConsumerMultiRecordEntryPointInterceptor(traceContext, descriptor, 0);
    interceptor.createTrace(new Object(), new Object[]{consumerRecords});

    verify(recorder).recordAcceptorHost("Unknown");
    verify(recorder).recordAttribute(KafkaConstants.KAFKA_TOPIC_ANNOTATION_KEY, "Test");
    verify(recorder).recordAttribute(KafkaConstants.KAFKA_BATCH_ANNOTATION_KEY, 1);
    verify(recorder).recordRpcName("kafka://topic=Test?batch=1");
}
 

public static void main(String[] args) {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:50088");
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "test_app");
    props.put(ConsumerConfig.CLIENT_ID_CONFIG, "test_app");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList("test_topic_0"));

    while (true) {
        ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000 * 1));
        for (ConsumerRecord<String, String> record : records) {
            System.out.println(String.format("record, key: %s, value: %s, offset: %s", record.key(), record.value(), record.offset()));
        }
    }
}
 

@Test
public void testTheTestHarness() {
  List<String> inputMessages = Arrays.asList("1", "2", "3", "4", "5", "6", "7", "8", "9", "10");
  // create input topic and produce the first batch of input messages
  boolean topicCreated = createTopic(INPUT_TOPIC, 1);
  if (!topicCreated) {
    fail("Could not create input topic.");
  }
  inputMessages.forEach(m -> produceMessage(INPUT_TOPIC, 0, m, m));

  // verify that the input messages were produced successfully
  if (inputMessages.size() > 0) {
    List<ConsumerRecord<String, String>> inputRecords =
        consumeMessages(Collections.singletonList(INPUT_TOPIC), inputMessages.size());
    List<String> readInputMessages = inputRecords.stream().map(ConsumerRecord::value).collect(Collectors.toList());
    Assert.assertEquals(inputMessages, readInputMessages);
  }
}
 

@Test
public void testConsumerBuilderWithStandardSpanNameProvider() throws InterruptedException {
  Producer<Integer, String> producer = createTracingProducer();
  producer.send(new ProducerRecord<>("messages", 1, "test"));
  producer.close();

  assertEquals(1, mockTracer.finishedSpans().size());

  ExecutorService executorService = Executors.newSingleThreadExecutor();
  final CountDownLatch latch = new CountDownLatch(1);

  executorService.execute(() -> {
    Consumer<Integer, String> consumer = createConsumerWithSpanNameProvider(null);

    while (latch.getCount() > 0) {
      ConsumerRecords<Integer, String> records = consumer.poll(Duration.ofMillis(100));
      for (ConsumerRecord<Integer, String> record : records) {
        SpanContext spanContext = TracingKafkaUtils
            .extractSpanContext(record.headers(), mockTracer);
        assertNotNull(spanContext);
        assertEquals("test", record.value());
        assertEquals((Integer) 1, record.key());

        consumer.commitSync();
        latch.countDown();
      }
    }
    consumer.close();
  });

  assertTrue(latch.await(30, TimeUnit.SECONDS));

  assertEquals("From_messages", mockTracer.finishedSpans().get(1).operationName());
}
 

@Advice.OnMethodExit(onThrowable = Throwable.class, suppress = Throwable.class)
public static void wrapIterable(@Nullable @Advice.Return(readOnly = false) Iterable<ConsumerRecord> iterable) {
    if (tracer == null || !tracer.isRunning() || tracer.currentTransaction() != null) {
        return;
    }

    //noinspection ConstantConditions,rawtypes
    KafkaInstrumentationHeadersHelper<ConsumerRecord, ProducerRecord> kafkaInstrumentationHelper =
        kafkaInstrHeadersHelperManager.getForClassLoaderOfClass(KafkaProducer.class);
    if (iterable != null && kafkaInstrumentationHelper != null) {
        iterable = kafkaInstrumentationHelper.wrapConsumerRecordIterable(iterable);
    }
}
 

private List<IngestionJobProto.FeatureSetSpecAck> getFeatureSetSpecAcks() {
  ConsumerRecords<String, IngestionJobProto.FeatureSetSpecAck> consumerRecords =
      consumer.poll(java.time.Duration.ofSeconds(KAFKA_POLL_TIMEOUT_SEC));

  return Lists.newArrayList(consumerRecords.records(KAFKA_SPECS_ACK_TOPIC)).stream()
      .map(ConsumerRecord::value)
      .collect(Collectors.toList());
}
 

@Test
public void shouldReturnSubpartitions() {

    // given
    SubpartitionSupplier<byte[], byte[]> subpartitionSupplier = new SubpartitionSupplier<>(new WorkerPartitioner<byte[], byte[]>() {

        @Override
        public int subpartition(ConsumerRecord<byte[], byte[]> consumerRecord) {
            return 1;
        }

        @Override
        public int count(TopicPartition topicPartition) {
            return topicPartition.partition();
        }
    });

    // when
    List<WorkerSubpartition> subpartitions = subpartitionSupplier
            .subpartitions(Arrays.asList(new TopicPartition("topic", 2), new TopicPartition("topic", 3)));

    // then
    assertThat(subpartitions.size()).isEqualTo(5);
    assertThat(subpartitions.get(0)).isEqualTo(WorkerSubpartition.getInstance("topic", 2, 0));
    assertThat(subpartitions.get(1)).isEqualTo(WorkerSubpartition.getInstance("topic", 2, 1));
    assertThat(subpartitions.get(2)).isEqualTo(WorkerSubpartition.getInstance("topic", 3, 0));
    assertThat(subpartitions.get(3)).isEqualTo(WorkerSubpartition.getInstance("topic", 3, 1));
    assertThat(subpartitions.get(4)).isEqualTo(WorkerSubpartition.getInstance("topic", 3, 2));
}
 

@Override
public User deserialize(ConsumerRecord<byte[], byte[]> input) {
  try {
    return mapper.readValue(input.value(), User.class);
  } catch (IOException e) {
    LOG.debug("Failed to deserialize record", e);
    return null;
  }
}
 

@Override
public void fail(Object msgId) {
    try {
        if (msgId != null && ConsumerRecord.class.isInstance(msgId)) {
            ConsumerRecord<String, byte[]> record = getMessageId(msgId);
            JSONObject wrapperJson = JSONObject.parseObject(new String(record.value()));
            String reqString = wrapperJson.getString(FullPullConstants.FULLPULL_REQ_PARAM);
            Long id = FullPullHelper.getSeqNo(reqString);
            String splitIndex = wrapperJson.getString(FullPullConstants.DATA_CHUNK_SPLIT_INDEX);
            logger.error("[pull fail] topic: {}, offset: {}, split index:{},  key: {}", record.topic(), record.offset(), splitIndex, record.key());

            shardsProcessManager.failAndClearShardElementQueue(id, record.offset());

            //写monitor，并且发送错误返回等， 只报错一次
            if (!failAndBreakTuplesSet.contains(reqString)) {
                FullPullHelper.finishPullReport(reqString, FullPullConstants.FULL_PULL_STATUS_ABORT, null);
            }
            sendFinishMsgToBolt(JSONObject.parseObject(reqString));
            failAndBreakTuplesSet.add(reqString);
        }
        super.fail(msgId);
    } catch (Exception e) {
        logger.error("[pull fail] exception!", e);
    }
}
 

private String processUmsMsg(EmitWarp<ConsumerRecord<String, byte[]>> data) throws Exception {
    DbusMessage ums = obtainUms(data);
    updateSchemaChangeFlag(data.getTableId());
    if (encodeConfigMap != null && encodeConfigMap.get(data.getTableId()) != null) {
        UmsEncoder encoder = new PluggableMessageEncoder(PluginManagerProvider.getManager(), (e, column, message) -> {
        });
        encoder.encode(ums, encodeConfigMap.get(data.getTableId()));
    } else {
        logger.debug("table id:{}, name space:{}, 脱敏配置信息为空,因此不执行脱敏.", data.getTableId(), data.getNameSpace());
    }
    return ums.toString();
}
 

@Override
public VerificationMessages.Command deserialize(ConsumerRecord<byte[], byte[]> input) {
  try {
    return VerificationMessages.Command.parseFrom(input.value());
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

@Test
public void queue_addRecord_executed() {
    ConsumerRecord record = new ConsumerRecord<>(topic, 0, 0, defaultKey, defaultValue);
    messageQueue.add(record);

    ArgumentCaptor<ConsumerRecord> captor = ArgumentCaptor.forClass(ConsumerRecord.class);
    verify(messageExecutor).execute(captor.capture());
    assertThat(captor.getValue()).isEqualTo(record);
}