org.apache.kafka.common.serialization.IntegerSerializer Java Examples

public SslProducer(String topic, int port) {
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost" + ":" + port);
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "DemoKafkaOnPulsarProducerSSL");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

    // SSL client config
    props.put("security.protocol", "SSL");
    props.put("ssl.truststore.location", "./src/test/resources/ssl/certificate/broker.truststore.jks");
    props.put("ssl.truststore.password", "broker");
    props.put("ssl.keystore.location", "./src/test/resources/ssl/certificate/client.keystore.jks");
    props.put("ssl.keystore.password", "client");

    // default is https, here need to set empty.
    props.put("ssl.endpoint.identification.algorithm", "");

    producer = new KafkaProducer<>(props);
    this.topic = topic;
}
 

@DataProvider(name = "serializers")
public Object[][] serializers() {
    return new Object[][] {
        {
            new StringSerializer(), StringDeserializer.class
        },
        {
            new LongSerializer(), LongDeserializer.class
        },
        {
            new IntegerSerializer(), IntegerDeserializer.class,
        },
        {
            new DoubleSerializer(), DoubleDeserializer.class,
        },
        {
            new BytesSerializer(), BytesDeserializer.class
        },
        {
            new ByteBufferSerializer(), ByteBufferDeserializer.class
        },
        {
            new ByteArraySerializer(), ByteArrayDeserializer.class
        }
    };
}
 

public static Properties loadProperties(String filename, String[] hosts, int port) {
    InputStream stream = null;

    try {
        Properties properties = new Properties();
        stream = KafkaBinaryLog.class.getClassLoader().getResourceAsStream(filename);
        properties.load(stream);

        String[] servers = new String[hosts.length];
        for (int i = 0; i < servers.length; i++)
            servers[i] = hosts[i] + ':' + port;

        properties.put("bootstrap.servers", StringUtils.join(servers, ','));
        properties.put("key.serializer", IntegerSerializer.class.getName());
        properties.put("value.serializer", KafkaPacketSerializer.class.getName());
        properties.put("key.deserializer", IntegerDeserializer.class.getName());
        properties.put("value.deserializer", KafkaPacketDeserializer.class.getName());

        return properties;
    } catch (IOException e) {
        throw new Error("Unexpected exception", e);
    } finally {
        IOUtils.closeQuietly(stream);
    }
}
 

public SslProducer(String topic, int port) {
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost" + ":" + port);
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "DemoKafkaOnPulsarProducerSSL");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

    // SSL client config
    props.put("security.protocol", "SSL");
    props.put("ssl.truststore.location", "./src/test/resources/ssl/certificate/broker.truststore.jks");
    props.put("ssl.truststore.password", "broker");

    // default is https, here need to set empty.
    props.put("ssl.endpoint.identification.algorithm", "");

    producer = new KafkaProducer<>(props);
    this.topic = topic;
}
 

static Deserializer getDeserializer(Serializer serializer) {
    if (serializer instanceof StringSerializer) {
        return new StringDeserializer();
    } else if (serializer instanceof LongSerializer) {
        return new LongDeserializer();
    } else if (serializer instanceof IntegerSerializer) {
        return new IntegerDeserializer();
    } else if (serializer instanceof DoubleSerializer) {
        return new DoubleDeserializer();
    } else if (serializer instanceof BytesSerializer) {
        return new BytesDeserializer();
    } else if (serializer instanceof ByteBufferSerializer) {
        return new ByteBufferDeserializer();
    } else if (serializer instanceof ByteArraySerializer) {
        return new ByteArrayDeserializer();
    } else {
        throw new IllegalArgumentException(serializer.getClass().getName() + " is not a valid or supported subclass of org.apache.kafka.common.serialization.Serializer.");
    }
}
 

public static void main(String[] args) {
    String topic = "persistent://public/default/test";

    Properties props = new Properties();
    props.put("bootstrap.servers", "pulsar://localhost:6650");
    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    Producer<Integer, String> producer = new KafkaProducer<>(props);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Integer, String>(topic, i, Integer.toString(i)));
        log.info("Message {} sent successfully", i);
    }

    producer.flush();
    producer.close();
}
 

public static void main(String[] args) {
    String topic = "persistent://public/default/test";

    Properties props = new Properties();
    props.put("bootstrap.servers", "pulsar://localhost:6650");

    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    Producer<Integer, String> producer = new KafkaProducer<>(props);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Integer, String>(topic, i, Integer.toString(i)));
        log.info("Message {} sent successfully", i);
    }

    producer.flush();
    producer.close();
}
 

private MapBasedConfig getDeadLetterQueueWithCustomConfig() {
    String prefix = "mp.messaging.incoming.kafka.";
    Map<String, Object> config = new HashMap<>();
    config.put(prefix + "connector", KafkaConnector.CONNECTOR_NAME);
    config.put(prefix + "group.id", "my-group");
    config.put(prefix + "topic", "dead-letter-custom");
    config.put(prefix + "value.deserializer", IntegerDeserializer.class.getName());
    config.put(prefix + "enable.auto.commit", "false");
    config.put(prefix + "auto.offset.reset", "earliest");
    config.put(prefix + "failure-strategy", "dead-letter-queue");
    config.put(prefix + "dead-letter-queue.topic", "missed");
    config.put(prefix + "dead-letter-queue.key.serializer", IntegerSerializer.class.getName());
    config.put(prefix + "dead-letter-queue.value.serializer", IntegerSerializer.class.getName());

    return new MapBasedConfig(config);
}
 

public static void main(String[] args) {

        logger.info("Creating Kafka Producer...");
        Properties props = new Properties();
        props.put(ProducerConfig.CLIENT_ID_CONFIG, AppConfigs.applicationID);
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

        KafkaProducer<Integer, String> producer = new KafkaProducer<>(props);

        logger.info("Start sending messages...");
        for (int i = 1; i <= AppConfigs.numEvents; i++) {
            producer.send(new ProducerRecord<>(AppConfigs.topicName, i, "Simple Message-" + i));
        }

        logger.info("Finished - Closing Kafka Producer.");
        producer.close();

    }
 

public static void main(String[] args) throws IOException {
		Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount-processor");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka0:19092");
        props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, "zookeeper0:12181/kafka");
        props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
		
		TopologyBuilder builder = new TopologyBuilder();
		builder.addSource("SOURCE", new StringDeserializer(), new StringDeserializer(), "words")
				.addProcessor("WordCountProcessor", WordCountProcessor::new, "SOURCE")
				.addStateStore(Stores.create("Counts").withStringKeys().withIntegerValues().inMemory().build(), "WordCountProcessor")
//				.connectProcessorAndStateStores("WordCountProcessor", "Counts")
				.addSink("SINK", "count", new StringSerializer(), new IntegerSerializer(), "WordCountProcessor");
		
        KafkaStreams stream = new KafkaStreams(builder, props);
        stream.start();
        System.in.read();
        stream.close();
        stream.cleanUp();
	}
 

/**
     * get kafkaProducer
     * Producer端的常用配置
     bootstrap.servers：Kafka集群连接串，可以由多个host:port组成
     acks：broker消息确认的模式，有三种：
     0：不进行消息接收确认，即Client端发送完成后不会等待Broker的确认
     1：由Leader确认，Leader接收到消息后会立即返回确认信息
     all：集群完整确认，Leader会等待所有in-sync的follower节点都确认收到消息后，再返回确认信息
     我们可以根据消息的重要程度，设置不同的确认模式。默认为1
     retries：发送失败时Producer端的重试次数，默认为0
     batch.size：当同时有大量消息要向同一个分区发送时，Producer端会将消息打包后进行批量发送。如果设置为0，则每条消息都独立发送。默认为16384字节
     linger.ms：发送消息前等待的毫秒数，与batch.size配合使用。在消息负载不高的情况下，配置linger.ms能够让Producer在发送消息前等待一定时间，以积累更多的消息打包发送，达到节省网络资源的目的。默认为0
     key.serializer/value.serializer：消息key/value的序列器Class，根据key和value的类型决定
     buffer.memory：消息缓冲池大小。尚未被发送的消息会保存在Producer的内存中，如果消息产生的速度大于消息发送的速度，那么缓冲池满后发送消息的请求会被阻塞。默认33554432字节（32MB）
     *
     *
     * @return
     */
    private static KafkaProducer<Integer, String> getProducer() {
        Properties properties = new Properties();
        //bootstrap.servers
//        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "123.56.118.135:9092,123.56.118.135:9093");
//        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "120.78.62.137:9093,120.78.62.137:9094");
        properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "120.78.62.137:9093,120.78.62.137:9094");
        //client.id
        properties.put(ProducerConfig.CLIENT_ID_CONFIG, PRODUCER_CLIENT_ID);
        //batch.size 当同时有大量消息要向同一个分区发送时，Producer端会将消息打包后进行批量发送。如果设置为0，则每条消息都独立发送。默认为16384字节
        properties.put(ProducerConfig.BATCH_SIZE_CONFIG,16384);
      //发送消息前等待的毫秒数，与batch.size配合使用。在消息负载不高的情况下，配置linger.ms能够让Producer在发送消息前等待一定时间，以积累更多的消息打包发送，达到节省网络资源的目的。默认为0
        properties.put(ProducerConfig.LINGER_MS_CONFIG,5000);
        //retries：发送失败时Producer端的重试次数，默认为0
        properties.put(ProducerConfig.RETRIES_CONFIG,0);
        //消息缓冲池大小。尚未被发送的消息会保存在Producer的内存中，如果消息产生的速度大于消息发送的速度，那么缓冲池满后发送消息的请求会被阻塞。默认33554432字节
        properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG,33554432);
        //key 和 value serializer的类
        properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
        properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
        return new KafkaProducer(properties);
    }
 

public static void main(String[] args) {
    String topic = Utils.getTopicName(args);

    Properties props = new Properties();
    props.put("bootstrap.servers", SERVICE_URL);
    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    Producer<Integer, String> producer = new KafkaProducer<>(props);

    log.info("Producer for topic {} successfully created", topic);

    IntStream.range(1, 101).forEach(i -> {
        String value = String.format("hello-%d", i);
        ProducerRecord<Integer, String> record = new ProducerRecord<>(topic, i, value);
        producer.send(record);
        log.info("Message with key {} sent successfully", record.key());
    });

    producer.close();
    log.info("Producer for topic {} successfully closed", topic);

    System.exit(0);
}
 

public void publishDummyDataNumbers() {
    final String topic = "NumbersTopic";

    // Create publisher
    final Map<String, Object> config = new HashMap<>();
    config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");

    final KafkaProducer<Integer, Integer> producer = new KafkaProducer<>(config);
    for (int value = 0; value < 10000; value++) {
        producer.send(new ProducerRecord<>(topic, value, value));
    }
    producer.flush();
    producer.close();
}
 

@Test
public void testSinkProducerRecordsWithCustomTS() throws Exception {
  int numElements = 1000;

  try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {

    ProducerSendCompletionThread completionThread =
        new ProducerSendCompletionThread(producerWrapper.mockProducer).start();

    final String defaultTopic = "test";
    final Long ts = System.currentTimeMillis();

    p.apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata())
        .apply(ParDo.of(new KV2ProducerRecord(defaultTopic, ts)))
        .setCoder(ProducerRecordCoder.of(VarIntCoder.of(), VarLongCoder.of()))
        .apply(
            KafkaIO.<Integer, Long>writeRecords()
                .withBootstrapServers("none")
                .withKeySerializer(IntegerSerializer.class)
                .withValueSerializer(LongSerializer.class)
                .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey)));

    p.run();

    completionThread.shutdown();

    // Verify that messages are written with user-defined timestamp
    List<ProducerRecord<Integer, Long>> sent = producerWrapper.mockProducer.history();

    for (int i = 0; i < numElements; i++) {
      ProducerRecord<Integer, Long> record = sent.get(i);
      assertEquals(defaultTopic, record.topic());
      assertEquals(i, record.key().intValue());
      assertEquals(i, record.value().longValue());
      assertEquals(ts, record.timestamp());
    }
  }
}
 

MockProducerWrapper() {
  producerKey = String.valueOf(ThreadLocalRandom.current().nextLong());
  mockProducer =
      new MockProducer<Integer, Long>(
          false, // disable synchronous completion of send. see ProducerSendCompletionThread
          // below.
          new IntegerSerializer(),
          new LongSerializer()) {

        // override flush() so that it does not complete all the waiting sends, giving a chance
        // to
        // ProducerCompletionThread to inject errors.

        @Override
        public synchronized void flush() {
          while (completeNext()) {
            // there are some uncompleted records. let the completion thread handle them.
            try {
              Thread.sleep(10);
            } catch (InterruptedException e) {
              // ok to retry.
            }
          }
        }
      };

  // Add the producer to the global map so that producer factory function can access it.
  assertNull(MOCK_PRODUCER_MAP.putIfAbsent(producerKey, mockProducer));
}
 

public static void main(String[] args) {
    String topic = "persistent://public/default/test-avro";

    Properties props = new Properties();
    props.put("bootstrap.servers", "pulsar://localhost:6650");

    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);


    Producer<Foo, Bar> producer = new KafkaProducer<>(props, fooSchema, barSchema);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Foo, Bar>(topic, i, foo, bar));
        log.info("Message {} sent successfully", i);
    }

    producer.flush();
    producer.close();
}
 

@Test
public void testSimpleProducer() throws Exception {
    String topic = "testSimpleProducer";

    @Cleanup
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(getPlainTextServiceUrl()).build();
    org.apache.pulsar.client.api.Consumer<byte[]> pulsarConsumer = pulsarClient.newConsumer().topic(topic)
            .subscriptionName("my-subscription")
            .subscribe();

    Properties props = new Properties();
    props.put("bootstrap.servers", getPlainTextServiceUrl());

    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    Producer<Integer, String> producer = new KafkaProducer<>(props);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Integer, String>(topic, i, "hello-" + i));
    }

    producer.flush();
    producer.close();

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = pulsarConsumer.receive(1, TimeUnit.SECONDS);
        assertEquals(new String(msg.getData()), "hello-" + i);
        pulsarConsumer.acknowledge(msg);
    }
}
 

public static void main(String[] args) {
    String topic = "persistent://public/default/test-avro";

    Properties props = new Properties();
    props.put("bootstrap.servers", "pulsar://localhost:6650");

    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    AvroSchema<Bar> barSchema = AvroSchema.of(SchemaDefinition.<Bar>builder().withPojo(Bar.class).build());
    AvroSchema<Foo> fooSchema = AvroSchema.of(SchemaDefinition.<Foo>builder().withPojo(Foo.class).build());

    Bar bar = new Bar();
    bar.setField1(true);

    Foo foo = new Foo();
    foo.setField1("field1");
    foo.setField2("field2");
    foo.setField3(3);


    Producer<Foo, Bar> producer = new KafkaProducer<>(props, fooSchema, barSchema);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Foo, Bar>(topic, i, foo, bar));
        log.info("Message {} sent successfully", i);
    }

    producer.flush();
    producer.close();
}
 

@BeforeTest
private void setup() {
    Properties producerProperties = new Properties();
    producerProperties.put("bootstrap.servers", getPlainTextServiceUrl());
    producerProperties.put("key.serializer", IntegerSerializer.class.getName());
    producerProperties.put("value.serializer", StringSerializer.class.getName());
    producer = new KafkaProducer<>(producerProperties);
}
 

@Test(timeOut = 10000)
public void testProducerCallback() throws Exception {
    String topic = "testProducerCallback";

    @Cleanup
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(getPlainTextServiceUrl()).build();
    org.apache.pulsar.client.api.Consumer<byte[]> pulsarConsumer = pulsarClient.newConsumer()
            .topic(topic)
            .subscriptionName("my-subscription")
            .subscribe();

    Properties props = new Properties();
    props.put("bootstrap.servers", getPlainTextServiceUrl());

    props.put("key.serializer", IntegerSerializer.class.getName());
    props.put("value.serializer", StringSerializer.class.getName());

    Producer<Integer, String> producer = new KafkaProducer<>(props);

    CountDownLatch counter = new CountDownLatch(10);

    for (int i = 0; i < 10; i++) {
        producer.send(new ProducerRecord<Integer, String>(topic, i, "hello-" + i), (metadata, exception) -> {
            assertEquals(metadata.topic(), topic);
            assertNull(exception);

            counter.countDown();
        });
    }

    counter.await();

    for (int i = 0; i < 10; i++) {
        Message<byte[]> msg = pulsarConsumer.receive(1, TimeUnit.SECONDS);
        assertEquals(new String(msg.getData()), "hello-" + i);
        pulsarConsumer.acknowledge(msg);
    }

    producer.close();
}
 

@Test
public void testStartAndStop() throws Exception {
    try (KafkaConsumer<Integer, String> consumer = new KafkaConsumer<>(cluster.consumerConfig(false), new IntegerDeserializer(), new StringDeserializer());
         KafkaProducer<Integer, String> producer = new KafkaProducer<>(cluster.producerConfig(), new IntegerSerializer(), new StringSerializer())) {
        cluster.createTopics(TEST_TOPIC);

        producer.send(new ProducerRecord<>(TEST_TOPIC, "value"));
        producer.flush();

        consumer.subscribe(Collections.singleton(TEST_TOPIC));
        ConsumerRecords<Integer, String> poll = consumer.poll(TEN_SECONDS);
        assertThat(poll.count()).isEqualTo(1);
        assertThat(poll.iterator().next().value()).isEqualTo("value");
    }
}
 

/**
 * 构造KafkaProducer
 *
 * @return KafkaProducer
 */
private KafkaProducer<Integer, String> getProducer() {
    Properties properties = new Properties();
    //bootstrap.servers
    properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "123.56.118.135:9092");
    //client.id
    properties.put(ProducerConfig.CLIENT_ID_CONFIG, "KafkaProducerTest");
    //key 和 value serializer的类
    properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    return new KafkaProducer(properties);
}
 

private void receiveAndValidateFoo(ConfigurableApplicationContext context) {
	Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
	DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(
			senderProps);
	KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
	template.setDefaultTopic("foos");
	template.sendDefault("{\"id\":\"123\"}");
	ConsumerRecord<String, String> cr = KafkaTestUtils.getSingleRecord(consumer,
			"counts-id");
	assertThat(cr.value().contains("Count for product with ID 123: 1")).isTrue();

	ProductCountApplication.Foo foo = context
			.getBean(ProductCountApplication.Foo.class);
	assertThat(foo.getProductStock(123).equals(1L));

	// perform assertions on HostInfo related methods in InteractiveQueryService
	InteractiveQueryService interactiveQueryService = context
			.getBean(InteractiveQueryService.class);
	HostInfo currentHostInfo = interactiveQueryService.getCurrentHostInfo();

	assertThat(currentHostInfo.host() + ":" + currentHostInfo.port())
			.isEqualTo(embeddedKafka.getBrokersAsString());

	HostInfo hostInfo = interactiveQueryService.getHostInfo("prod-id-count-store",
			123, new IntegerSerializer());
	assertThat(hostInfo.host() + ":" + hostInfo.port())
			.isEqualTo(embeddedKafka.getBrokersAsString());

	HostInfo hostInfoFoo = interactiveQueryService
			.getHostInfo("prod-id-count-store-foo", 123, new IntegerSerializer());
	assertThat(hostInfoFoo).isNull();

	final List<HostInfo> hostInfos = interactiveQueryService.getAllHostsInfo("prod-id-count-store");
	assertThat(hostInfos.size()).isEqualTo(1);
	final HostInfo hostInfo1 = hostInfos.get(0);
	assertThat(hostInfo1.host() + ":" + hostInfo1.port())
			.isEqualTo(embeddedKafka.getBrokersAsString());

}
 

@Test
public void integerSerializer_integerObject() {
	KafkaPublishTask kPublishTask = new KafkaPublishTask(new SystemPropertiesConfiguration(), new KafkaProducerManager(new SystemPropertiesConfiguration()), objectMapper);
	KafkaPublishTask.Input input = new KafkaPublishTask.Input();
	input.setKeySerializer(IntegerSerializer.class.getCanonicalName());
	input.setKey(String.valueOf(Integer.MAX_VALUE));
	Assert.assertEquals(kPublishTask.getKey(input), new Integer(Integer.MAX_VALUE));
}
 

@VisibleForTesting
Object getKey(Input input) {

	String keySerializer = input.getKeySerializer();

	if (LongSerializer.class.getCanonicalName().equals(keySerializer)) {
		return Long.parseLong(String.valueOf(input.getKey()));
	} else if (IntegerSerializer.class.getCanonicalName().equals(keySerializer)) {
		return Integer.parseInt(String.valueOf(input.getKey()));
	} else {
		return String.valueOf(input.getKey());
	}

}
 

private Map<String, Object> senderProperties() {
    Map<String, Object> props = new HashMap<>();

    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, info.getHosts());
    props.put(ProducerConfig.RETRIES_CONFIG, 0);
    props.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384);
    props.put(ProducerConfig.LINGER_MS_CONFIG, 1);
    props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432);
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class);
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);

    return props;
}
 

public static Producer<Integer, String> createProducer() {
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, System.getProperty("camel.component.kafka.brokers"));
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "test-consumer");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());

    return new KafkaProducer<>(props);
}
 

@Override
public Iterator<String> call(Integer partition, Iterator<T> locatedRowIterator) throws Exception {
    taskContext = TaskContext.get();

    if (taskContext != null && taskContext.attemptNumber() > 0) {
        LOG.trace("KS.c attempts "+taskContext.attemptNumber());
        long entriesInKafka = KafkaUtils.messageCount(bootstrapServers, topicName, partition);
        LOG.trace("KS.c entries "+entriesInKafka);
        for (long i = 0; i < entriesInKafka; ++i) {
            locatedRowIterator.next();
        }
    }

    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "spark-producer-dss-ks-"+UUID.randomUUID() );
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ExternalizableSerializer.class.getName());
    KafkaProducer<Integer, Externalizable> producer = new KafkaProducer<>(props);
    int count = 0 ;
    while (locatedRowIterator.hasNext()) {
        T lr = locatedRowIterator.next();

        ProducerRecord<Integer, Externalizable> record = new ProducerRecord(topicName, count++, lr);
        producer.send(record);
        LOG.trace("KS.c sent "+partition.intValue()+" "+count+" "+lr);
    }
    LOG.trace("KS.c count "+partition.intValue()+" "+count);

    producer.close();
    // TODO Clean up
    return Arrays.asList("OK").iterator();
}
 

public static Producer<Integer, String> createProducer() {
    Properties props = new Properties();
    props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:19093");
    props.put(ProducerConfig.CLIENT_ID_CONFIG, "test-ssl-producer");
    props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName());
    props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName());
    addSsl(props);

    return new KafkaProducer<>(props);
}
 

@Test
public void testSink() throws Exception {
  // Simply read from kafka source and write to kafka sink. Then verify the records
  // are correctly published to mock kafka producer.

  int numElements = 1000;

  try (MockProducerWrapper producerWrapper = new MockProducerWrapper()) {

    ProducerSendCompletionThread completionThread =
        new ProducerSendCompletionThread(producerWrapper.mockProducer).start();

    String topic = "test";

    p.apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata())
        .apply(
            KafkaIO.<Integer, Long>write()
                .withBootstrapServers("none")
                .withTopic(topic)
                .withKeySerializer(IntegerSerializer.class)
                .withValueSerializer(LongSerializer.class)
                .withInputTimestamp()
                .withProducerFactoryFn(new ProducerFactoryFn(producerWrapper.producerKey)));

    p.run();

    completionThread.shutdown();

    verifyProducerRecords(producerWrapper.mockProducer, topic, numElements, false, true);
  }
}