org.apache.kafka.clients.producer.Producer Java Examples
The following examples show how to use
org.apache.kafka.clients.producer.Producer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaFactory.java From nakadi with MIT License | 6 votes |
@Nullable private Producer<String, String> takeUnderLock(final boolean canCreate) { final Lock lock = canCreate ? rwLock.writeLock() : rwLock.readLock(); lock.lock(); try { if (null != activeProducer) { useCount.get(activeProducer).incrementAndGet(); return activeProducer; } else if (canCreate) { activeProducer = createProducerInstance(); useCount.put(activeProducer, new AtomicInteger(1)); LOG.info("New producer instance created: " + activeProducer); return activeProducer; } else { return null; } } finally { lock.unlock(); } }
Example #2
Source File: AsynchronousDeliveryStrategy.java From logback-kafka-appender with Apache License 2.0 | 6 votes |
@Override public <K, V, E> boolean send(Producer<K, V> producer, ProducerRecord<K, V> record, final E event, final FailedDeliveryCallback<E> failedDeliveryCallback) { try { producer.send(record, new Callback() { @Override public void onCompletion(RecordMetadata metadata, Exception exception) { if (exception != null) { failedDeliveryCallback.onFailedDelivery(event, exception); } } }); return true; } catch (BufferExhaustedException | TimeoutException e) { failedDeliveryCallback.onFailedDelivery(event, e); return false; } }
Example #3
Source File: KafkaRyaStreamsClientFactory.java From rya with Apache License 2.0 | 6 votes |
/** * Create a {@link Producer} that is able to write to a topic in Kafka. * * @param kafkaHostname - The Kafka broker hostname. (not null) * @param kafkaPort - The Kafka broker port. * @param keySerializerClass - Serializes the keys. (not null) * @param valueSerializerClass - Serializes the values. (not null) * @return A {@link Producer} that can be used to write records to a topic. */ private static <K, V> Producer<K, V> makeProducer( final String kafkaHostname, final int kakfaPort, final Class<? extends Serializer<K>> keySerializerClass, final Class<? extends Serializer<V>> valueSerializerClass) { requireNonNull(kafkaHostname); requireNonNull(keySerializerClass); requireNonNull(valueSerializerClass); final Properties producerProps = new Properties(); producerProps.setProperty(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, kafkaHostname + ":" + kakfaPort); producerProps.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySerializerClass.getName()); producerProps.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializerClass.getName()); return new KafkaProducer<>(producerProps); }
Example #4
Source File: LiKafkaProducerFactory.java From brooklin with BSD 2-Clause "Simplified" License | 6 votes |
@Override public Producer<byte[], byte[]> createProducer(Properties transportProps) { VerifiableProperties transportProviderProperties = new VerifiableProperties(transportProps); String clientId = transportProviderProperties.getString(ProducerConfig.CLIENT_ID_CONFIG); String bootstrapServers = transportProviderProperties.getString(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG); Properties producerConfig = transportProviderProperties.getDomainProperties(DOMAIN_PRODUCER); Validate.notEmpty(clientId, "clientId cannot be empty."); Validate.notEmpty(bootstrapServers, "bootstrapServers cannot be empty."); producerConfig = buildProducerProperties(producerConfig, clientId, bootstrapServers, DEFAULT_ENABLE_LARGE_MESSAGE); // Default DeSerializer for Key and Payload producerConfig.putIfAbsent(LiKafkaProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); producerConfig.putIfAbsent(LiKafkaProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); return new LiKafkaProducerImpl<>(producerConfig); }
Example #5
Source File: KafkaProducerDemo.java From kafka_book_demo with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws ExecutionException, InterruptedException{ Properties properties = new Properties(); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList); properties.put(ProducerConfig.CLIENT_ID_CONFIG, "spark-producer-demo-client"); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); Producer<String,String> producer = new KafkaProducer<>(properties); Random random = new Random(); while (true) { int value = random.nextInt(10); ProducerRecord<String, String> message = new ProducerRecord<>(topic, value+""); producer.send(message, (recordMetadata, e) -> { if (recordMetadata != null) { System.out.println(recordMetadata.topic() + "-" + recordMetadata.partition() + ":" + recordMetadata.offset()); } }); TimeUnit.SECONDS.sleep(1); } }
Example #6
Source File: ProducerInTransaction.java From javatech with Creative Commons Attribution Share Alike 4.0 International | 6 votes |
public static Producer buildProducer() { // 1. 指定生产者的配置 Properties properties = new Properties(); properties.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, HOST); properties.put(ProducerConfig.ACKS_CONFIG, "all"); properties.put(ProducerConfig.RETRIES_CONFIG, 1); properties.put(ProducerConfig.BATCH_SIZE_CONFIG, 16384); properties.put(ProducerConfig.LINGER_MS_CONFIG, 1); properties.put(ProducerConfig.BUFFER_MEMORY_CONFIG, 33554432); properties.put(ProducerConfig.TRANSACTIONAL_ID_CONFIG, "first-transactional"); properties.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); properties.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer"); // 2. 使用配置初始化 Kafka 生产者 Producer<String, String> producer = new KafkaProducer<>(properties); return producer; }
Example #7
Source File: KafkaNotificationMockTest.java From atlas with Apache License 2.0 | 6 votes |
@Test @SuppressWarnings("unchecked") public void shouldThrowExceptionIfProducerFails() throws NotificationException, ExecutionException, InterruptedException { Properties configProperties = mock(Properties.class); KafkaNotification kafkaNotification = new KafkaNotification(configProperties); Producer producer = mock(Producer.class); String topicName = kafkaNotification.getProducerTopicName(NotificationInterface.NotificationType.HOOK); String message = "This is a test message"; Future returnValue = mock(Future.class); when(returnValue.get()).thenThrow(new RuntimeException("Simulating exception")); ProducerRecord expectedRecord = new ProducerRecord(topicName, message); when(producer.send(expectedRecord)).thenReturn(returnValue); try { kafkaNotification.sendInternalToProducer(producer, NotificationInterface.NotificationType.HOOK, Arrays.asList(new String[]{message})); fail("Should have thrown NotificationException"); } catch (NotificationException e) { assertEquals(e.getFailedMessages().size(), 1); assertEquals(e.getFailedMessages().get(0), "This is a test message"); } }
Example #8
Source File: KafkaSupport.java From brooklyn-library with Apache License 2.0 | 6 votes |
/** * Send a message to the {@link KafkaCluster} on the given topic. */ public void sendMessage(String topic, String message) { Optional<Entity> anyBrokerNodeInCluster = Iterables.tryFind(cluster.getCluster().getChildren(), Predicates.and( Predicates.instanceOf(KafkaBroker.class), EntityPredicates.attributeEqualTo(KafkaBroker.SERVICE_UP, true))); if (anyBrokerNodeInCluster.isPresent()) { KafkaBroker broker = (KafkaBroker)anyBrokerNodeInCluster.get(); Properties props = new Properties(); props.put("metadata.broker.list", format("%s:%d", broker.getAttribute(KafkaBroker.HOSTNAME), broker.getKafkaPort())); props.put("bootstrap.servers", format("%s:%d", broker.getAttribute(KafkaBroker.HOSTNAME), broker.getKafkaPort())); props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); Producer<String, String> producer = new KafkaProducer<>(props); ((KafkaZooKeeper)cluster.getZooKeeper()).createTopic(topic); ProducerRecord<String, String> data = new ProducerRecord<>(topic, message); producer.send(data); producer.close(); } else { throw new InvalidParameterException("No kafka broker node found"); } }
Example #9
Source File: KafkaServiceImpl.java From kafka-eagle with Apache License 2.0 | 6 votes |
/** * Send mock message to kafka topic . */ public boolean mockMessage(String clusterAlias, String topic, String message) { Properties props = new Properties(); props.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, getKafkaBrokerServer(clusterAlias)); props.put(Kafka.KEY_SERIALIZER, StringSerializer.class.getCanonicalName()); props.put(Kafka.VALUE_SERIALIZER, StringSerializer.class.getCanonicalName()); props.put(Kafka.PARTITION_CLASS, KafkaPartitioner.class.getName()); if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.sasl.enable")) { sasl(props, clusterAlias); } if (SystemConfigUtils.getBooleanProperty(clusterAlias + ".kafka.eagle.ssl.enable")) { ssl(props, clusterAlias); } Producer<String, String> producer = new KafkaProducer<>(props); producer.send(new ProducerRecord<String, String>(topic, new Date().getTime() + "", message)); producer.close(); return true; }
Example #10
Source File: KafkaDatasetOtherDelimTestIT.java From components with Apache License 2.0 | 6 votes |
@Before public void init() throws TimeoutException { // there may exists other topics than these build in(configured in pom.xml) topics, but ignore them // ----------------- Send sample data to TOPIC_IN start -------------------- String testID = "sampleTest" + new Random().nextInt(); List<Person> expectedPersons = Person.genRandomList(testID, 10); Properties props = new Properties(); props.put("bootstrap.servers", BOOTSTRAP_HOST); props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); Producer<Void, String> producer = new KafkaProducer<>(props); for (Person person : expectedPersons) { ProducerRecord<Void, String> message = new ProducerRecord<>(TOPIC_IN, person.toCSV(fieldDelimiter)); producer.send(message); } producer.close(); // ----------------- Send sample data to TOPIC_IN end -------------------- }
Example #11
Source File: FlinkKafkaProducer.java From flink with Apache License 2.0 | 6 votes |
private static int[] getPartitionsByTopic(String topic, Producer<byte[], byte[]> producer) { // the fetched list is immutable, so we're creating a mutable copy in order to sort it List<PartitionInfo> partitionsList = new ArrayList<>(producer.partitionsFor(topic)); // sort the partitions by partition id to make sure the fetched partition list is the same across subtasks Collections.sort(partitionsList, new Comparator<PartitionInfo>() { @Override public int compare(PartitionInfo o1, PartitionInfo o2) { return Integer.compare(o1.partition(), o2.partition()); } }); int[] partitions = new int[partitionsList.size()]; for (int i = 0; i < partitions.length; i++) { partitions[i] = partitionsList.get(i).partition(); } return partitions; }
Example #12
Source File: RunQueryCommandIT.java From rya with Apache License 2.0 | 6 votes |
@Before public void setup() { // Make sure the topic that the change log uses exists. final String changeLogTopic = KafkaTopics.queryChangeLogTopic("" + ryaInstance); kafka.createTopic(changeLogTopic); // Setup the QueryRepository used by the test. final Producer<?, QueryChange> queryProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, QueryChangeSerializer.class); final Consumer<?, QueryChange> queryConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, QueryChangeDeserializer.class); final QueryChangeLog changeLog = new KafkaQueryChangeLog(queryProducer, queryConsumer, changeLogTopic); queryRepo = new InMemoryQueryRepository(changeLog, Scheduler.newFixedRateSchedule(0L, 5, TimeUnit.SECONDS)); // Initialize the Statements Producer and the Results Consumer. stmtProducer = KafkaTestUtil.makeProducer(kafka, StringSerializer.class, VisibilityStatementSerializer.class); resultConsumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, VisibilityBindingSetDeserializer.class); }
Example #13
Source File: DefaultProducerFactoryTest.java From extension-kafka with Apache License 2.0 | 6 votes |
@Test void testTransactionalProducerCreation() { assumeFalse( System.getProperty("os.name").contains("Windows"), "Transactional producers not supported on Windows" ); ProducerFactory<String, String> producerFactory = transactionalProducerFactory(kafkaBroker, "xyz"); Producer<String, String> testProducer = producerFactory.createProducer(); testProducer.beginTransaction(); testProducer.commitTransaction(); assertFalse(testProducer.metrics().isEmpty()); cleanup(producerFactory, testProducer); }
Example #14
Source File: ProduceData.java From oryx with Apache License 2.0 | 6 votes |
public void start() throws InterruptedException { RandomGenerator random = RandomManager.getRandom(); Properties props = ConfigUtils.keyValueToProperties( "bootstrap.servers", "localhost:" + kafkaPort, "key.serializer", "org.apache.kafka.common.serialization.StringSerializer", "value.serializer", "org.apache.kafka.common.serialization.StringSerializer", "compression.type", "gzip", "linger.ms", 0, "batch.size", 0, "acks", 1, "max.request.size", 1 << 26 // TODO ); try (Producer<String,String> producer = new KafkaProducer<>(props)) { for (int i = 0; i < howMany; i++) { Pair<String,String> datum = datumGenerator.generate(i, random); ProducerRecord<String,String> record = new ProducerRecord<>(topic, datum.getFirst(), datum.getSecond()); producer.send(record); log.debug("Sent datum {} = {}", record.key(), record.value()); if (intervalMsec > 0) { Thread.sleep(intervalMsec); } } } }
Example #15
Source File: MessageProducerFactory.java From alcor with Apache License 2.0 | 6 votes |
public Producer Create() { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, this.kafkaAddress); props.put(ProducerConfig.CLIENT_ID_CONFIG, IKafkaConfiguration.PRODUCER_CLIENT_ID); // Key is set as long and Value is given by concrete implementation props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, LongSerializer.class.getName()); Serializer serializer = getSerializer(); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, serializer.getClass().getName()); //TODO: Optimizing partition // props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, CustomPartitioner.class.getName()); return new KafkaProducer<>(props); }
Example #16
Source File: BoltCommandHandlerHelper.java From DBus with Apache License 2.0 | 6 votes |
public static void writeEmailMessage(String subject, String contents, String dataSchema, Producer<String, String> producer) { try { // 发邮件 ControlMessage gm = new ControlMessage(System.currentTimeMillis(), ControlType.COMMON_EMAIL_MESSAGE.toString(), BoltCommandHandlerHelper.class.getName()); gm.addPayload("subject", subject); gm.addPayload("contents", contents); gm.addPayload("datasource_schema", Utils.getDatasource().getDsName() + "/" + dataSchema); String topic = PropertiesHolder.getProperties(Constants.Properties.CONFIGURE, Constants.ConfigureKey.GLOBAL_EVENT_TOPIC); ProducerRecord<String, String> record = new ProducerRecord<>(topic, gm.getType(), gm.toJSONString()); producer.send(record, (metadata, exception) -> { if (exception != null) { logger.error("Send global event error.{}", exception.getMessage()); } }); } catch (Exception e) { logger.error("send email error. schema:{}, subject:{}, content:{}", dataSchema, subject, contents, e); } finally { if (producer != null) producer.close(); } }
Example #17
Source File: KafkaAvroSerDesWithKafkaServerTest.java From registry with Apache License 2.0 | 6 votes |
private String produceMessage(String topicName, Object msg, Boolean storeSchemaInHeader) { String bootstrapServers = CLUSTER.bootstrapServers(); Map<String, Object> config = new HashMap<>(); config.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); config.putAll(SCHEMA_REGISTRY_TEST_SERVER_CLIENT_WRAPPER.exportClientConf(true)); config.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()); config.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, KafkaAvroSerializer.class.getName()); config.put(KafkaAvroSerializer.STORE_SCHEMA_VERSION_ID_IN_HEADER, storeSchemaInHeader.toString()); final Producer<String, Object> producer = new KafkaProducer<>(config); final Callback callback = new ProducerCallback(); LOG.info("Sending message: [{}] to topic: [{}]", msg, topicName); ProducerRecord<String, Object> producerRecord = new ProducerRecord<>(topicName, getKey(msg), msg); producer.send(producerRecord, callback); producer.flush(); LOG.info("Message successfully sent to topic: [{}]", topicName); producer.close(5, TimeUnit.SECONDS); return bootstrapServers; }
Example #18
Source File: KafkaClientMetricsProducerTest.java From micrometer with Apache License 2.0 | 5 votes |
@Test void shouldCreateMetersWithTags() { try (Producer<String, String> producer = createProducer()) { metrics = new KafkaClientMetrics(producer, tags); MeterRegistry registry = new SimpleMeterRegistry(); metrics.bindTo(registry); assertThat(registry.getMeters()) .hasSizeGreaterThan(0) .extracting(meter -> meter.getId().getTag("app")) .allMatch(s -> s.equals("myapp")); } }
Example #19
Source File: OnrampServiceConfiguration.java From data-highway with Apache License 2.0 | 5 votes |
@SuppressWarnings("deprecation") @Bean public Producer<byte[], byte[]> kafkaProducer( @Value("${kafka.bootstrapServers}") String bootstrapServers, @Value("${kafka.road.batch.size:16384}") int batchSize, @Value("${kafka.road.linger.ms:0}") int lingerMs, @Value("${kafka.road.buffer.memory:33554432}") long bufferMemory, @Value("${kafka.road.acks:1}") String acks, @Value("${kafka.road.compression:none}") String compressionType, @Value("${kafka.producer.request.timeout.ms:10000}") int requestTimeout, MeterRegistry registry) { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); props.put(ProducerConfig.ACKS_CONFIG, acks); props.put(ProducerConfig.RETRIES_CONFIG, 100); props.put(ProducerConfig.BATCH_SIZE_CONFIG, batchSize); props.put(ProducerConfig.LINGER_MS_CONFIG, lingerMs); props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, bufferMemory); props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, compressionType); props.put(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, requestTimeout); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getCanonicalName()); Producer<byte[], byte[]> producer = new KafkaProducer<>(props); producer.metrics().forEach((metricName, metric) -> { String name = "onramp_kafka_producer_" + metricName.group() + "_" + metricName.name(); registry.gauge(name, metric, m -> m.value()); }); return producer; }
Example #20
Source File: PublisherPool.java From nifi with Apache License 2.0 | 5 votes |
private PublisherLease createLease() { final Producer<byte[], byte[]> producer = new KafkaProducer<>(kafkaProperties); final PublisherLease lease = new PublisherLease(producer, maxMessageSize, maxAckWaitMillis, logger) { @Override public void close() { if (isPoisoned() || isClosed()) { super.close(); } else { publisherQueue.offer(this); } } }; return lease; }
Example #21
Source File: KafkaPublisherTest.java From extension-kafka with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static DefaultProducerFactory<String, byte[]> producerFactoryWithFencedExceptionOnAbort() { DefaultProducerFactory<String, byte[]> producerFactory = mock(DefaultProducerFactory.class, "FactoryForExceptionOnAbortTx"); Producer<String, byte[]> producer = mock(Producer.class, "ExceptionOnAbortTx"); when(producerFactory.confirmationMode()).thenReturn(ConfirmationMode.TRANSACTIONAL); when(producerFactory.createProducer()).thenReturn(producer); doThrow(RuntimeException.class).when(producer).abortTransaction(); return producerFactory; }
Example #22
Source File: CamelKafkaResource.java From camel-quarkus with Apache License 2.0 | 5 votes |
@Path("/kafka/{topicName}") @POST @Produces(MediaType.APPLICATION_JSON) public JsonObject post(@PathParam("topicName") String topicName, String message) throws Exception { try (Producer<Integer, String> producer = CamelKafkaSupport.createProducer()) { RecordMetadata meta = producer.send(new ProducerRecord<>(topicName, 1, message)).get(); return Json.createObjectBuilder() .add("topicName", meta.topic()) .add("partition", meta.partition()) .add("offset", meta.offset()) .build(); } }
Example #23
Source File: KafkaContainer.java From DBus with Apache License 2.0 | 5 votes |
public Producer getProducer(Properties props) { if (producerMap.containsKey(props)) { return producerMap.get(props); } else { Producer producer = new KafkaProducer<>(props); producerMap.put(props, producer); return producer; } }
Example #24
Source File: AlarmConfigProducerDemo.java From phoebus with Eclipse Public License 1.0 | 5 votes |
private void sendItemRemoval(final Producer<String, AlarmTreeItem<?>> producer, final String topic, final String path) { final String key = AlarmSystem.CONFIG_PREFIX + path; final ProducerRecord<String, AlarmTreeItem<?>> record = new ProducerRecord<>(topic, partition, key, null); producer.send(record); }
Example #25
Source File: KafkaRpcClientConfig.java From devicehive-java-server with Apache License 2.0 | 5 votes |
@Profile("!test") @Bean(destroyMethod = "shutdown") public RpcClient rpcClient(Producer<String, Request> requestProducer, RequestResponseMatcher responseMatcher, ServerResponseListener responseListener) { KafkaRpcClient client = new KafkaRpcClient(REQUEST_TOPIC, RESPONSE_TOPIC, requestProducer, responseMatcher, responseListener); client.start(); return client; }
Example #26
Source File: KafkaProducerManager.java From quarkus with Apache License 2.0 | 5 votes |
public static Producer<Integer, String> createProducer() { Properties props = new Properties(); props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:19092"); props.put(ProducerConfig.CLIENT_ID_CONFIG, "test"); props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, IntegerSerializer.class.getName()); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); return new KafkaProducer<Integer, String>(props); }
Example #27
Source File: TestProducer.java From java-study with Apache License 2.0 | 5 votes |
public static void main(String[] args) { System.out.println("开始..."); Properties props = new Properties(); props.put("bootstrap.servers", "192.169.0.23:9092"); //The "all" setting we have specified will result in blocking on the full commit of the record, the slowest but most durable setting. //“所有”设置将导致记录的完整提交阻塞,最慢的,但最持久的设置。 props.put("acks", "all"); //如果请求失败,生产者也会自动重试,即使设置成0 the producer can automatically retry. props.put("retries", 0); //The producer maintains buffers of unsent records for each partition. props.put("batch.size", 16384); //默认立即发送,这里这是延时毫秒数 props.put("linger.ms", 1); //生产者缓冲大小,当缓冲区耗尽后,额外的发送调用将被阻塞。时间超过max.block.ms将抛出TimeoutException props.put("buffer.memory", 33554432); //The key.serializer and value.serializer instruct how to turn the key and value objects the user provides with their ProducerRecord into bytes. props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); //创建kafka的生产者类 Producer<String, String> producer = new KafkaProducer<String, String>(props); long startTime=System.currentTimeMillis(); producer.send(new ProducerRecord<String, String>("test1",1,startTime,"a","b")); producer.close(); //生产者的主要方法 // close();//Close this producer. // close(long timeout, TimeUnit timeUnit); //This method waits up to timeout for the producer to complete the sending of all incomplete requests. // flush() ;所有缓存记录被立刻发送 // for(int i = 0; i < 100; i++){ // //这里平均写入4个分区 // producer.send(new ProducerRecord<String, String>("foo",i%4, Integer.toString(i), Integer.toString(i))); // producer.close(); // } System.out.println("结束"); }
Example #28
Source File: KafkaTestUtil.java From rya with Apache License 2.0 | 5 votes |
/** * Create a {@link Producer} that is able to write to a topic that is hosted within an embedded instance of Kafka. * * @param kafka - The Kafka rule used to connect to the embedded Kafka instance. (not null) * @param keySerializerClass - Serializes the keys. (not null) * @param valueSerializerClass - Serializes the values. (not null) * @return A {@link Producer} that can be used to write records to a topic. */ public static <K, V> Producer<K, V> makeProducer( final KafkaTestInstanceRule kafka, final Class<? extends Serializer<K>> keySerializerClass, final Class<? extends Serializer<V>> valueSerializerClass) { requireNonNull(kafka); requireNonNull(keySerializerClass); requireNonNull(valueSerializerClass); final Properties props = kafka.createBootstrapServerConfig(); props.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, keySerializerClass.getName()); props.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, valueSerializerClass.getName()); return new KafkaProducer<>(props); }
Example #29
Source File: DefaultKafkaClientFactory.java From kbear with Apache License 2.0 | 5 votes |
@Override public <K, V> Producer<K, V> newProducer(Map<String, Object> configs, Serializer<K> keySerializer, Serializer<V> valueSerializer) { ObjectExtension.requireNonNull(configs, "configs"); Properties properties = new Properties(); properties.putAll(configs); return newProducer(properties, keySerializer, valueSerializer); }
Example #30
Source File: KafkaProducerPoolTest.java From common-kafka with Apache License 2.0 | 5 votes |
@Test public void sameConfiguration() { Properties props = KafkaTests.getProps(); props.setProperty(KAFKA_PRODUCER_CONCURRENCY, "1"); props.setProperty(KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); props.setProperty(VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class.getName()); Producer<String, String> p1 = pool.getProducer(props); Producer<String, String> p2 = pool.getProducer(props); assertThat(p1, is(p2)); }