Java Code Examples for org.apache.kafka.clients.producer.KafkaProducer#close()

public void publishDummyData() {
    final String topic = "TestTopic";

    // Create publisher
    final Map<String, Object> config = new HashMap<>();
    config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
    config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");

    final KafkaProducer<String, String> producer = new KafkaProducer<>(config);
    for (int charCode = 65; charCode < 91; charCode++) {
        final char[] key = new char[1];
        key[0] = (char) charCode;

        producer.send(new ProducerRecord<>(topic, new String(key), new String(key)));
    }
    producer.flush();
    producer.close();
}
 

/**
 * 向kafka发送批量消息
 * @param listMsg 发送的消息
 * @param url 发送的地址
 * @param topicName 消息名称
 * @return
 * @throws Exception 
 */
public static boolean sendMessage(List<String> listMsg,String url,String topicName) {
	KafkaProducer<String, String> producer=null;
	boolean falg=false;
	try{
		Properties props=init(url);
		producer= new KafkaProducer<String, String>(props);
		for(String msg:listMsg){
			producer.send(new ProducerRecord<String, String>(topicName,msg));
		}
		falg=true;
	}catch(Exception e){
		e.printStackTrace();
	}finally{
		producer.close();
	}
	return falg;
}
 

private synchronized void closeProducer(KafkaProducer<?, ?> producer, boolean fromCallback) {
    try {
        if (producer == null) producer = this.producer;
        if (producer != null && producer == this.producer) {
            try {
                log.info("Closing resilient producer.");
                if (fromCallback) {
                    producer.close(Duration.ZERO);
                } else {
                    producer.close();
                }
            } catch (Exception e) {
                log.warn("Exception caught while closing producer.", e);
            } finally {
                this.producer = null;
            }
        }
    } finally {
        if (!fromCallback) closed = true;
    }
}
 

/**
 * 向kafka发送批量消息
 * @param listMsg 发送的消息
 * @param url 发送的地址
 * @param topicName 消息名称
 * @return
 * @throws Exception 
 */
public static boolean sendMessage(List<String> listMsg,String url,String topicName) {
	KafkaProducer<String, String> producer=null;
	boolean flag=false;
	try{
		Properties props=init(url);
		producer= new KafkaProducer<String, String>(props);
		for(String msg:listMsg){
			producer.send(new ProducerRecord<String, String>(topicName,msg));
		}
		flag=true;
	}catch(Exception e){
		e.printStackTrace();
	}finally{
		if(producer!=null){
			producer.close();
		}
	}
	return flag;
}
 

/**
 * Topic topicName will be automatically created if it doesn't exist.
 * @param topicName
 * @param recordsToPublish
 * @param timestamp
 * @return
 * @throws InterruptedException
 * @throws TimeoutException
 * @throws ExecutionException
 */
public Map<String, RecordMetadata> produceData(String topicName,
                                               Map<String, GenericRow> recordsToPublish,
                                               Serializer<GenericRow> serializer,
                                               Long timestamp)
        throws InterruptedException, TimeoutException, ExecutionException {

  createTopic(topicName);

  Properties producerConfig = properties();
  KafkaProducer<String, GenericRow> producer =
          new KafkaProducer<>(producerConfig, new StringSerializer(), serializer);

  Map<String, RecordMetadata> result = new HashMap<>();
  for (Map.Entry<String, GenericRow> recordEntry : recordsToPublish.entrySet()) {
    String key = recordEntry.getKey();
    Future<RecordMetadata> recordMetadataFuture
        = producer.send(buildRecord(topicName, timestamp, recordEntry, key));
    result.put(key,
               recordMetadataFuture.get(TEST_RECORD_FUTURE_TIMEOUT_MS, TimeUnit.MILLISECONDS));
  }
  producer.close();

  return result;
}
 

public static void main(String[] args) {

        logger.info("Creating Kafka Producer...");
        Properties props = new Properties();
        props.put(ProducerConfig.CLIENT_ID_CONFIG, AppConfigs.applicationID);
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

        KafkaProducer<Integer, String> producer = new KafkaProducer<>(props);

        logger.info("Start sending messages...");
        for (int i = 1; i <= AppConfigs.numEvents; i++) {
            producer.send(new ProducerRecord<>(AppConfigs.topicName, i, "Simple Message-" + i));
        }

        logger.info("Finished - Closing Kafka Producer.");
        producer.close();

    }
 

public void publishDummyDataNumbers() {
    final String topic = "NumbersTopic";

    // Create publisher
    final Map<String, Object> config = new HashMap<>();
    config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");

    final KafkaProducer<Integer, Integer> producer = new KafkaProducer<>(config);
    for (int value = 0; value < 10000; value++) {
        producer.send(new ProducerRecord<>(topic, value, value));
    }
    producer.flush();
    producer.close();
}
 

public static void main(String[] args) throws ExecutionException, InterruptedException {
  /** TODO: 设置 Producer 属性 */
  Properties properties = new Properties();
  /** TODO: Kafka 服务地址 */
  properties.put(
      ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "node-160:9092,node-161:9092,node-162:9092");
  /** TODO: 设置 key 序列化类 */
  properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
  /** TODO: 设置 value 序列化类 */
  properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

  /** TODO: 创建 Kafka Producer */
  KafkaProducer<String, String> producer = new KafkaProducer<>(properties);

  /** TODO: 同步发送消息 */
  RecordMetadata recordMetadata =
      producer.send(new ProducerRecord<>("topic01", "Hello Kafka")).get();
  System.out.println(recordMetadata);

  /** TODO: 关闭 producer */
  producer.close();
}
 

public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put("key.serializer",
                "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("value.serializer",
                "org.apache.kafka.common.serialization.StringSerializer");
        properties.put("bootstrap.servers", brokerList);


        KafkaProducer<String, String> producer =
                new KafkaProducer<>(properties);
        ProducerRecord<String, String> record =
                new ProducerRecord<>(topic, "hello, Kafka!");
        try {
            producer.send(record);
//            producer.send(record).get();
        } catch (Exception e) {
            e.printStackTrace();
        }
        producer.close();
    }
 

@Override
public void run() {
    PropertyReader propertyReader = new PropertyReader();

    Properties producerProps = new Properties();
    producerProps.put("bootstrap.servers", propertyReader.getPropertyValue("broker.list"));
    producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("auto.create.topics.enable", "true");

    KafkaProducer<String, String> ipProducer = new KafkaProducer<String, String>(producerProps);

    BufferedReader br = readFile();
    String oldLine = "";
    try {
        while ((oldLine = br.readLine()) != null) {
            String line = getNewRecordWithRandomIP(oldLine).replace("[", "").replace("]", "");
            ProducerRecord ipData = new ProducerRecord<String, String>(propertyReader.getPropertyValue("topic"), line);
            Future<RecordMetadata> recordMetadata = ipProducer.send(ipData);
        }
    } catch (IOException e) {
        e.printStackTrace();
    }
    ipProducer.close();
}
 

/**
 * Reset the kafka producer
 * @param config  the kafka producer config
 * @return true if the reset operation succeed, otherwise return false.
 */
private boolean resetProducerInternal(KafkaProducerConfig config) {
  boolean retval = false;
  KafkaProducer<byte[], byte[]> oldProducer = producers.get(config);
  if (oldProducer != null) {
    oldProducer.close();
    KafkaProducer<byte[], byte[]> newProducer = KafkaUtils.createKafkaProducer(config);
    retval = producers.replace(config, oldProducer, newProducer);
    if (!retval) {
      newProducer.close();
    }
    // log metrics for no.of kafka producers currently in the cache
    OpenTsdbMetricConverter.addMetric(SingerMetrics.NUM_KAFKA_PRODUCERS, producers.size());
  }
  return retval;
}
 

@Test
public void testAvroConsumer() {
    KafkaProducer<Integer, Pet> producer = createProducer();
    Pet pet = new Pet();
    pet.setName("neo");
    pet.setColor("white");
    producer.send(new ProducerRecord<>("test-avro-consumer", 1, pet));
    Pet retrieved = RestAssured.when().get("/avro").as(Pet.class);
    Assertions.assertEquals("neo", retrieved.getName());
    Assertions.assertEquals("white", retrieved.getColor());
    producer.close();
}
 

@Test
public void testPublishMetrics() {
  KafkaProducerConfig config = new KafkaProducerConfig("/var/serverset/discovery.kafka.prod",
      Arrays.asList("localhost:9092"), "-1");
  KafkaProducerManager.getInstance().getProducers().clear();
  KafkaProducer<byte[], byte[]> producer = KafkaProducerManager.getProducer(config);
  KafkaProducerMetricsMonitor monitor = new KafkaProducerMetricsMonitor();
  monitor.publishKafkaProducerMetricsToOstrich();
  producer.close();
  for (String metricName : KafkaProducerMetricsMonitor.PRODUCER_METRICS_WHITELIST) {
    Object gauge = Stats.getGauge("kafkaproducer." + metricName + " cluster=kafka").get();
    assertNotNull(gauge);
  }
}
 

@Override
public void shutdown() {
    LOGGER.info("start closing kafka producer, broker cluster:{}", brokerCluster);
    for (KafkaProducer<String, byte[]> kafkaProducer : kafkaProducers) {
        kafkaProducer.close();
    }
}
 

@SuppressWarnings( {"rawtypes", "unchecked"})
@Ignore
@Test
public void generateRandomStringsToKafka() {
    String topic = "testTopic3";
    int max = 1000;
    Properties configMap = new Properties();
    configMap.put("bootstrap.servers", "localhost:6667");
    configMap.put("metadata.broker.list", "localhost:6667");
    configMap.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    configMap.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    configMap.put("request.required.acks", "1");
    configMap.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    configMap.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    KafkaProducer<String, Object> producer = new KafkaProducer<>(configMap);

    int i = 0;
    while (i++ < max) {
        String randomString = generateRandomString();
        System.out.println("sending string : " + randomString);
        ProducerRecord record = new ProducerRecord(topic, randomString);
        producer.send(record);
        if (i % 10 == 0) {
            try {
                Thread.sleep(10);
            } catch (Exception ex) {
            }
        }
    }
    producer.close();
}
 

/**
 * Applications entry point
 *
 * @param args topicName (name of the Kafka topic) numEvents (# of messages)
 */
public static void main(String[] args) {
    String topicName;
    int numEvents;

    if (args.length != 2) {
        System.out.println("Please provide command line arguments: topicName numEvents");
        System.exit(-1);
    }
    topicName = args[0];
    numEvents = Integer.valueOf(args[1]);
    logger.info("Starting HelloProducer...");
    logger.debug("topicName=" + topicName + ", numEvents=" + numEvents);

    logger.trace("Creating Kafka Producer...");
    Properties props = new Properties();
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "HelloProducer");
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

    KafkaProducer<Integer, String> producer = new KafkaProducer<>(props);

    logger.trace("Start sending messages...");
    try {
        for (int i = 1; i <= numEvents; i++) {
            producer.send(new ProducerRecord<>(topicName, i, "Simple Message-" + i));
        }
    } catch (KafkaException e) {
        logger.error("Exception occurred - Check log for more details.\n" + e.getMessage());
        System.exit(-1);
    } finally {
        logger.info("Finished HelloProducer - Closing Kafka Producer.");
        producer.close();
    }

}
 

/**
 * Test kafka without rya code to make sure kafka works in this environment.
 * If this test fails then its a testing environment issue, not with Rya.
 * Source: https://github.com/asmaier/mini-kafka
 */
@Test
public void embeddedKafkaTest() throws Exception {
    // create topic
    final String topic = "testTopic";
    AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

    // setup producer
    final Properties producerProps = new Properties();
    producerProps.setProperty("bootstrap.servers", BROKERHOST + ":" + BROKERPORT);
    producerProps.setProperty("key.serializer", "org.apache.kafka.common.serialization.IntegerSerializer");
    producerProps.setProperty("value.serializer", "org.apache.kafka.common.serialization.ByteArraySerializer");
    final KafkaProducer<Integer, byte[]> producer = new KafkaProducer<>(producerProps);

    // setup consumer
    final Properties consumerProps = new Properties();
    consumerProps.setProperty("bootstrap.servers", BROKERHOST + ":" + BROKERPORT);
    consumerProps.setProperty("group.id", "group0");
    consumerProps.setProperty("client.id", "consumer0");
    consumerProps.setProperty("key.deserializer", "org.apache.kafka.common.serialization.IntegerDeserializer");
    consumerProps.setProperty("value.deserializer", "org.apache.kafka.common.serialization.ByteArrayDeserializer");

    // to make sure the consumer starts from the beginning of the topic
    consumerProps.put("auto.offset.reset", "earliest");

    final KafkaConsumer<Integer, byte[]> consumer = new KafkaConsumer<>(consumerProps);
    consumer.subscribe(Arrays.asList(topic));

    // send message
    final ProducerRecord<Integer, byte[]> data = new ProducerRecord<>(topic, 42, "test-message".getBytes(StandardCharsets.UTF_8));
    producer.send(data);
    producer.close();

    // starting consumer
    final ConsumerRecords<Integer, byte[]> records = consumer.poll(3000);
    assertEquals(1, records.count());
    final Iterator<ConsumerRecord<Integer, byte[]>> recordIterator = records.iterator();
    final ConsumerRecord<Integer, byte[]> record = recordIterator.next();
    assertEquals(42, (int) record.key());
    assertEquals("test-message", new String(record.value(), StandardCharsets.UTF_8));
    consumer.close();
}
 

@Override public synchronized void close() {
  if (closeCalled) return;
  KafkaProducer<byte[], byte[]>  producer = this.producer;
  if (producer != null) producer.close();
  closeCalled = true;
}
 

@Test
public void testNative() throws IOException, InterruptedException, ExecutionException {
	final String topic = "kafka.SimpleAvroTopic";
	int msgNum = 200;
	final CountDownLatch countDown = new CountDownLatch(msgNum);

	Properties producerProps = new Properties();
	producerProps.put("bootstrap.servers", "");

	// Avro Decoder/Encoder
	CachedSchemaRegistryClient schemaRegistry = new CachedSchemaRegistryClient("",
	      AbstractKafkaAvroSerDeConfig.MAX_SCHEMAS_PER_SUBJECT_DEFAULT);
	Map<String, String> configs = new HashMap<String, String>();
	configs.put("schema.registry.url", "");

	KafkaAvroSerializer avroKeySerializer = new KafkaAvroSerializer();
	avroKeySerializer.configure(configs, true);
	KafkaAvroSerializer avroValueSerializer = new KafkaAvroSerializer();
	avroValueSerializer.configure(configs, false);

	Map<String, String> deserializerConfigs = new HashMap<String, String>();
	deserializerConfigs.put("specific.avro.reader", Boolean.TRUE.toString());
	deserializerConfigs.put("schema.registry.url", "");
	KafkaAvroDeserializer avroKeyDeserializer = new KafkaAvroDeserializer(schemaRegistry, deserializerConfigs);
	avroKeyDeserializer.configure(configs, true);
	KafkaAvroDeserializer avroValueDeserializer = new KafkaAvroDeserializer(schemaRegistry, deserializerConfigs);
	avroValueDeserializer.configure(configs, false);

	// Consumer
	final Properties consumerProps = new Properties();
	consumerProps.put("bootstrap.servers", "");
	consumerProps.put("group.id", "GROUP_" + topic);

	final List<Object> actualResult = new ArrayList<Object>();
	final List<Object> expectedResult = new ArrayList<Object>();

	final KafkaConsumer<Object, Object> consumer = new KafkaConsumer<Object, Object>(consumerProps,
	      avroKeyDeserializer, avroValueDeserializer);
	consumer.subscribe(Arrays.asList(topic));

	class KafkaConsumerThread implements Runnable {

		private final AtomicBoolean closed = new AtomicBoolean(false);

		public void run() {
			try {
				while (!closed.get()) {
					ConsumerRecords<Object, Object> records = consumer.poll(100);
					for (ConsumerRecord<Object, Object> consumerRecord : records) {
						System.out.println("received: " + consumerRecord.value());
						actualResult.add(consumerRecord.value());
						countDown.countDown();
					}
				}
			} catch (WakeupException e) {
				if (!closed.get())
					throw e;
			} finally {
				consumer.commitSync();
				consumer.close();
			}
		}

		public void shutdown() {
			closed.set(true);
			consumer.wakeup();
		}
	}

	KafkaConsumerThread thread = new KafkaConsumerThread();
	new Thread(thread).start();

	KafkaProducer<Object, Object> producer = new KafkaProducer<Object, Object>(producerProps, avroKeySerializer,
	      avroValueSerializer);
	int i = 0;
	while (i++ < msgNum) {
		ProducerRecord<Object, Object> data = new ProducerRecord<Object, Object>(topic, null,
		      (Object) KafkaAvroTest.generateEvent());
		Future<RecordMetadata> send = producer.send(data);
		send.get();
		if (send.isDone()) {
			System.out.println("sending: " + data.value());
			expectedResult.add(data.value());
		}
	}

	countDown.await();

	thread.shutdown();
	producer.close();

	Assert.assertEquals(expectedResult.size(), actualResult.size());
}
 

@Override
public String call() throws Exception {
    // set up the producer
    KafkaProducer<String, String> producer;
    try (InputStream props = Resources.getResource("producer.props").openStream()) {
        Properties properties = new Properties();
        properties.load(props);
        producer = new KafkaProducer<>(properties);
    }

    try {
        String request;

        while (keepRunning.get()) {
            while ((request = requestQueue.poll()) != null) {
                // simple class containing only the type
                Message message = gson.fromJson(request, Message.class);
                String topic = null;

                switch (message.getType()) {
                    case "TickerCommand":
                        topic = "stock-ticker-cmd";
                        break;
                    case "UpDownCommand":
                        topic = "updown-cmd";
                        break;
                    case "WeatherCommand":
                        topic = "weather-cmd";
                        break;
                }

                if (topic != null) {
                    producer.send(new ProducerRecord<>(topic, request),
                            (metadata, e) -> {
                                if (e != null) {
                                    logger.error(e.getMessage(), e);
                                }

                                logger.debug("The offset of the record we just sent is: " + metadata.offset());
                            });
                } else {
                    logger.info(String.format("Ignoring message with unknown type %s", message.getType()));
                }
            }

            producer.flush();
            Thread.sleep(20L);
        }
    } catch (Throwable throwable) {
        logger.error(throwable.getMessage(), throwable);
    } finally {
        producer.close();
    }

    return "done";
}