Java Code Examples for org.apache.kafka.clients.consumer.KafkaConsumer

The following examples show how to use org.apache.kafka.clients.consumer.KafkaConsumer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: pulsar   Source File: ConsumerExample.java    License: Apache License 2.0 8 votes vote down vote up
public static void main(String[] args) {
    String topic = "persistent://public/default/test";

    Properties props = new Properties();
    props.put("bootstrap.servers", "pulsar://localhost:6650");
    props.put("group.id", "my-subscription-name");
    props.put("enable.auto.commit", "false");
    props.put("key.deserializer", IntegerDeserializer.class.getName());
    props.put("value.deserializer", StringDeserializer.class.getName());

    @SuppressWarnings("resource")
    Consumer<Integer, String> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList(topic));

    while (true) {
        ConsumerRecords<Integer, String> records = consumer.poll(100);
        records.forEach(record -> {
            log.info("Received record: {}", record);
        });

        // Commit last offset
        consumer.commitSync();
    }
}
 
Example 2
public static void main(String[] args) {
	// 1. 指定消费者的配置
	final Properties props = new Properties();
	props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, HOST);
	props.put(ConsumerConfig.GROUP_ID_CONFIG, "test");
	props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
	props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
	props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
		"org.apache.kafka.common.serialization.StringDeserializer");
	props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
		"org.apache.kafka.common.serialization.StringDeserializer");

	// 2. 使用配置初始化 Kafka 消费者
	KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);

	// 3. 消费者订阅 Topic
	consumer.subscribe(Arrays.asList("t1"));
	while (true) {
		// 4. 消费消息
		ConsumerRecords<String, String> records = consumer.poll(100);
		for (ConsumerRecord<String, String> record : records) {
			System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
		}
	}
}
 
Example 3
Source Project: kylin-on-parquet-v2   Source File: KafkaClient.java    License: Apache License 2.0 6 votes vote down vote up
public static Map<Integer, Long> getLatestOffsets(final CubeInstance cubeInstance) {
    final KafkaConfig kafkaConfig = KafkaConfigManager.getInstance(KylinConfig.getInstanceFromEnv()).getKafkaConfig(cubeInstance.getRootFactTable());

    final String brokers = KafkaClient.getKafkaBrokers(kafkaConfig);
    final String topic = kafkaConfig.getTopic();

    Map<Integer, Long> startOffsets = Maps.newHashMap();
    try (final KafkaConsumer consumer = KafkaClient.getKafkaConsumer(brokers, cubeInstance.getName())) {
        final List<PartitionInfo> partitionInfos = consumer.partitionsFor(topic);
        for (PartitionInfo partitionInfo : partitionInfos) {
            long latest = getLatestOffset(consumer, topic, partitionInfo.partition());
            startOffsets.put(partitionInfo.partition(), latest);
        }
    }
    return startOffsets;
}
 
Example 4
Source Project: synapse   Source File: SynapseKafkaAutoConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
@Bean
@ConditionalOnMissingBean(name = "kafkaMessageLogReceiverEndpointFactory")
public MessageLogReceiverEndpointFactory kafkaMessageLogReceiverEndpointFactory(final KafkaProperties kafkaProperties,
                                                                                final MessageInterceptorRegistry interceptorRegistry,
                                                                                final ApplicationEventPublisher eventPublisher,
                                                                                final ConsumerFactory<String, String> kafkaConsumerFactory) {
    LOG.info("Auto-configuring Kafka MessageLogReceiverEndpointFactory");
    final ExecutorService executorService = newCachedThreadPool(
            new ThreadFactoryBuilder().setNameFormat("kafka-message-log-%d").build()
    );

    final KafkaConsumer<String, String> kafkaConsumer = (KafkaConsumer<String, String>)kafkaConsumerFactory.createConsumer();

    return new KafkaMessageLogReceiverEndpointFactory(
            interceptorRegistry,
            kafkaConsumer,
            executorService,
            eventPublisher);
}
 
Example 5
Source Project: kafka-serializer-example   Source File: Example.java    License: MIT License 6 votes vote down vote up
public static void runConsumer(Properties properties, String topic) throws Exception {
    properties.put("group.id", "test");
    properties.put("enable.auto.commit", "true");
    properties.put("auto.commit.interval.ms", "1000");
    properties.put("session.timeout.ms", "30000");
    properties.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

    System.out.printf("Running consumer with serializer %s on topic %s\n", properties.getProperty("value.deserializer"), topic);

    KafkaConsumer<String, SensorReading> consumer = new KafkaConsumer<>(properties);
    consumer.subscribe(Arrays.asList(topic));
    while (true) {
        ConsumerRecords<String, SensorReading> records = consumer.poll(100);
        for (ConsumerRecord<String, SensorReading> record : records)
            System.out.printf("offset = %d, key = %s, value = %s\n", record.offset(), record.key(), record.value());
    }
}
 
Example 6
/**
 * Return Kafka Consumer configured to consume from internal Kafka Server.
 * @param <K> Type of message key
 * @param <V> Type of message value
 * @param keyDeserializer Class of deserializer to be used for keys.
 * @param valueDeserializer Class of deserializer to be used for values.
 * @param config Additional consumer configuration options to be set.
 * @return KafkaProducer configured to produce into Test server.
 */
public <K, V> KafkaConsumer<K, V> getKafkaConsumer(
    final Class<? extends Deserializer<K>> keyDeserializer,
    final Class<? extends Deserializer<V>> valueDeserializer,
    final Properties config
) {

    // Build config
    final Map<String, Object> kafkaConsumerConfig = buildDefaultClientConfig();
    kafkaConsumerConfig.put("key.deserializer", keyDeserializer);
    kafkaConsumerConfig.put("value.deserializer", valueDeserializer);
    kafkaConsumerConfig.put("partition.assignment.strategy", "org.apache.kafka.clients.consumer.RoundRobinAssignor");

    // Override config
    if (config != null) {
        for (final Map.Entry<Object, Object> entry: config.entrySet()) {
            kafkaConsumerConfig.put(entry.getKey().toString(), entry.getValue());
        }
    }

    // Create and return Consumer.
    return new KafkaConsumer<>(kafkaConsumerConfig);
}
 
Example 7
Source Project: BigData-In-Practice   Source File: CheckBeginingOffset.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    KafkaConsumer<String, String> kafkaConsumer = createNewConsumer();
    List<PartitionInfo> partitions = kafkaConsumer.partitionsFor("topic-monitor");
    List<TopicPartition> tpList = partitions.stream()
            .map(pInfo -> new TopicPartition(pInfo.topic(), pInfo.partition()))
            .collect(toList());
    Map<TopicPartition, Long> beginningOffsets =
            kafkaConsumer.beginningOffsets(tpList);
    System.out.println(beginningOffsets);
}
 
Example 8
Source Project: SkyEye   Source File: Launcher.java    License: GNU General Public License v3.0 6 votes vote down vote up
public static void main(String[] args) {
    SpringApplicationBuilder builder = new SpringApplicationBuilder(Launcher.class);
    Set<ApplicationListener<?>> listeners = builder.application().getListeners();
    for (Iterator<ApplicationListener<?>> it = listeners.iterator(); it.hasNext();) {
        ApplicationListener<?> listener = it.next();
        if (listener instanceof LoggingApplicationListener) {
            it.remove();
        }
    }
    builder.application().setListeners(listeners);
    ConfigurableApplicationContext context = builder.run(args);
    LOGGER.info("collector metrics start successfully");

    KafkaConsumer kafkaConsumer = (KafkaConsumer<byte[], String>) context.getBean("kafkaConsumer");
    Task task = (Task) context.getBean("metricsTask");

    // 优雅停止项目
    Runtime.getRuntime().addShutdownHook(new ShutdownHookRunner(kafkaConsumer, task));
    task.doTask();
}
 
Example 9
Source Project: common-kafka   Source File: KafkaAdminClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void getConsumerGroupSummary() {
    client.createTopic(testName.getMethodName(), 1, 1);

    Properties properties = new Properties();
    properties.putAll(KafkaTests.getProps());
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, BytesDeserializer.class.getName());
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, testName.getMethodName());
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, testName.getMethodName() + "-client-id");

    try (Consumer<Object, Object> consumer = new KafkaConsumer<>(properties)) {
        consumer.subscribe(Arrays.asList(testName.getMethodName()));
        consumer.poll(Duration.ofSeconds(5L));

        AdminClient.ConsumerGroupSummary summary = client.getConsumerGroupSummary(testName.getMethodName());
        assertThat("Expected only 1 consumer summary when getConsumerGroupSummaries(" + testName.getMethodName() + ")",
                convertToJavaSet(summary.consumers().get().iterator()).size(), is(1));

        assertThat(summary.state(), is(notNullValue()));
        assertThat(summary.coordinator(), is(notNullValue()));
        assertThat(summary.assignmentStrategy(), is(notNullValue()));
    }
}
 
Example 10
public KafkaTopicBroadcaster(String name, ObjectMapper objectMapper, String zookeeper) {
    super();
    Properties props = new Properties();
    try {
        props.put("client.id", InetAddress.getLocalHost().getHostName());
    } catch (UnknownHostException e) {
        throw new RuntimeException();
    }
    props.put("bootstrap.servers", zookeeper);
    props.put("group.id", name);
    props.put("key.deserializer", StringDeserializer.class);
    props.put("value.deserializer", StringDeserializer.class);
    props.put("enable.auto.commit", "false");
    props.put("auto.offset.reset", "earliest");

    this.consumer = new KafkaConsumer(props);
    this.objectMapper = objectMapper;
}
 
Example 11
Source Project: BigData-In-Practice   Source File: OffsetCommitSyncBatch.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    KafkaConsumer<String, String> consumer = new ConsumerFactory<String, String>().create();
    final int minBatchSize = 200;
    List<ConsumerRecord> buffer = new ArrayList<>();
    while (true) {
        ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
        for (ConsumerRecord<String, String> record : records) {
            buffer.add(record);
            System.out.println(record.offset() + " : " + record.value());
        }
        if (buffer.size() >= minBatchSize) {
            //do some logical processing with buffer.
            consumer.commitSync();
            buffer.clear();
        }
    }
}
 
Example 12
@PostConstruct
public void startConsuming() {
  consumerThread = new Thread("kafka-workaround-consumer") {
    public void run( ) {
      
      final Properties props = new Properties();
      props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
      props.put(ConsumerConfig.GROUP_ID_CONFIG, consumerGroup);
      props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
      props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

      consumer = new KafkaConsumer<>(props);
      consumer.subscribe(Collections.singletonList(topicName));
      while (running) {
        consumer.poll(pollingInterval);
        consumer.commitAsync();
      }        
      consumer.close();
    }
  };
  consumerThread.start();
}
 
Example 13
Source Project: nifi   Source File: AtlasNotificationServerEmulator.java    License: Apache License 2.0 6 votes vote down vote up
public void consume(Consumer<HookNotification> c) {
    Properties props = new Properties();
    props.put("bootstrap.servers", "localhost:9092");
    props.put("group.id", "test");
    props.put("enable.auto.commit", "true");
    props.put("auto.commit.interval.ms", "1000");
    props.put("session.timeout.ms", "30000");
    props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList("ATLAS_HOOK"));

    isStopped = false;
    while (!isStopped) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        for (ConsumerRecord<String, String> record : records) {
            final MessageDeserializer deserializer = NotificationInterface.NotificationType.HOOK.getDeserializer();
            final HookNotification m
                    = (HookNotification) deserializer.deserialize(record.value());
            c.accept(m);
        }
    }

    consumer.close();
}
 
Example 14
public static void main(String[] args) {
    KafkaConsumer<String, String> consumer = createConsumer();
    consumer.subscribe(Arrays.asList(TOPIC));

    boolean flag = true;


    while (true) {
        ConsumerRecords<String, String> records = consumer.poll(100);
        if (flag) {
            Set<TopicPartition> assignments = consumer.assignment();
            assignments.forEach(topicPartition ->
                    consumer.seek(
                            topicPartition,
                            90));
            flag = false;
        }


        for (ConsumerRecord<String, String> record : records)
            System.out.printf("offset = %d, key = %s, value = %s%n", record.offset(), record.key(), record.value());
    }


}
 
Example 15
public KafkaEasyTransMsgConsumerImpl(ConsumerConfig cfg, ObjectSerializer serializer,
		KafkaEasyTransMsgPublisherImpl retryQueueMsgProducer) {
	this.serializer = serializer;
	this.cfg = cfg;
	consumer = new KafkaConsumer<>(cfg.getNativeCfg());
	reconsumer = new KafkaConsumer<>(cfg.getNativeCfg());
	this.retryQueueMsgProducer = retryQueueMsgProducer;
	threadPool = Executors.newFixedThreadPool(cfg.getConsumerThread(), new NamedThreadFactory("KafkaMsgHandler"));

	// 计算每个重试次数对应的重试时间等级阈值
	List<List<Integer>> reconsumeCfg = cfg.getReconsume();
	initRetryThreshold(reconsumeCfg);
	initRetryRecordsMap();
	initRetryQueueSubscribe(reconsumeCfg);
	initRetryQueuePartitionCountMap();
}
 
Example 16
Source Project: vertx-kafka-client   Source File: EarliestNativeTest.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {

    Map<String, String> config = new HashMap<>();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    config.put(ConsumerConfig.GROUP_ID_CONFIG, "my-group");
    config.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
    config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

    KafkaConsumer consumer = new KafkaConsumer(config);
    consumer.subscribe(Collections.singleton("my-topic"));

    while (true) {
      ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
      for (ConsumerRecord<String, String> record: records) {
        System.out.println(record);
      }
    }
  }
 
Example 17
Source Project: kafka-webview   Source File: ParallelWebKafkaConsumer.java    License: MIT License 6 votes vote down vote up
private List<KafkaResult> consume(final KafkaConsumer kafkaConsumer) {
    final List<KafkaResult> kafkaResultList = new ArrayList<>();
    final ConsumerRecords<?,?> consumerRecords = kafkaConsumer.poll(pollTimeoutDuration);

    logger.info("Consumed {} records", consumerRecords.count());
    for (final ConsumerRecord consumerRecord : consumerRecords) {
        // Get next record
        // Convert to KafkaResult.
        final KafkaResult kafkaResult = new KafkaResult(
            consumerRecord.partition(),
            consumerRecord.offset(),
            consumerRecord.timestamp(),
            consumerRecord.key(),
            consumerRecord.value()
        );

        // Add to list.
        kafkaResultList.add(kafkaResult);
    }

    // Commit offsets
    commit(kafkaConsumer);
    return kafkaResultList;
}
 
Example 18
Source Project: DBus   Source File: MaasAppenderEvent.java    License: Apache License 2.0 6 votes vote down vote up
public MaasAppenderEvent(String topic, String dataTopic) {
    super(01);
    this.topic = topic;
    this.dataTopic = dataTopic;
    dao = new DbusDataDaoImpl();
    Properties props = HeartBeatConfigContainer.getInstance().getKafkaConsumerConfig();
    Properties producerProps = HeartBeatConfigContainer.getInstance().getmaasConf().getProducerProp();
    try {
        LoggerFactory.getLogger().info("[topic]   ...." + topic);
        LoggerFactory.getLogger().info("[maas-appender-event]  initial.........................");
        dataConsumer = new KafkaConsumer<>(props);
        partition0 = new TopicPartition(this.topic, 0);
        dataConsumer.assign(Arrays.asList(partition0));
        dataConsumer.seekToEnd(Arrays.asList(partition0));

        statProducer = new KafkaProducer<>(producerProps);

    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 19
Source Project: kafka-graphs   Source File: StreamUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
private static <K, V> List<KeyValue<K, V>> consumeData(
    String topic,
    Class keyDeserializer,
    Class valueDeserializer,
    int expectedNumMessages,
    long resultsPollMaxTimeMs) {

    List<KeyValue<K, V>> result = new ArrayList<>();

    Properties consumerConfig = ClientUtils.consumerConfig(CLUSTER.bootstrapServers(), "testgroup",
        keyDeserializer, valueDeserializer, new Properties());
    try (KafkaConsumer<K, V> consumer = new KafkaConsumer<>(consumerConfig)) {

        consumer.subscribe(Collections.singleton(topic));
        long pollStart = System.currentTimeMillis();
        long pollEnd = pollStart + resultsPollMaxTimeMs;
        while (System.currentTimeMillis() < pollEnd &&
            continueConsuming(result.size(), expectedNumMessages)) {
            for (ConsumerRecord<K, V> record :
                consumer.poll(Duration.ofMillis(Math.max(1, pollEnd - System.currentTimeMillis())))) {
                if (record.value() != null) {
                    result.add(new KeyValue<>(record.key(), record.value()));
                }
            }
        }
    }
    return result;
}
 
Example 20
Source Project: ja-micro   Source File: PartitionAssignmentWatchdog.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized void subscriberInitialized(KafkaConsumer<String, String> realConsumer) {
    logger.debug("Adding subscriber");
    consumers.add(realConsumer);
    if (consumers.size() == 1) {
        startWatchdog();
    }
}
 
Example 21
Source Project: kylin-on-parquet-v2   Source File: KafkaSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String getMessageTemplate(StreamingSourceConfig streamingSourceConfig) {
    String template = null;
    KafkaConsumer<byte[], byte[]> consumer = null;
    try {
        String topicName = getTopicName(streamingSourceConfig.getProperties());
        Map<String, Object> config = getKafkaConf(streamingSourceConfig.getProperties());
        consumer = new KafkaConsumer<>(config);
        Set<TopicPartition> partitions = Sets.newHashSet(FluentIterable.from(consumer.partitionsFor(topicName))
                .transform(new Function<PartitionInfo, TopicPartition>() {
                    @Override
                    public TopicPartition apply(PartitionInfo input) {
                        return new TopicPartition(input.topic(), input.partition());
                    }
                }));
        consumer.assign(partitions);
        consumer.seekToBeginning(partitions);
        ConsumerRecords<byte[], byte[]> records = consumer.poll(500);
        if (records == null) {
            return null;
        }
        Iterator<ConsumerRecord<byte[], byte[]>> iterator = records.iterator();
        if (iterator == null || !iterator.hasNext()) {
            return null;
        }
        ConsumerRecord<byte[], byte[]> record = iterator.next();
        template = new String(record.value(), "UTF8");
    } catch (Exception e) {
        logger.error("error when fetch one record from kafka, stream:" + streamingSourceConfig.getName(), e);
    } finally {
        if (consumer != null) {
            consumer.close();
        }
    }
    return template;
}
 
Example 22
Source Project: data-highway   Source File: RoadEndpointsIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
private static KafkaConsumer<String, String> createPatchConsumer() {
  Properties properties = new Properties();
  properties.setProperty("bootstrap.servers", kafkaCluster.bootstrapServers());
  properties.setProperty("group.id", UUID.randomUUID().toString());
  properties.setProperty("auto.offset.reset", "earliest");
  properties.setProperty("enable.auto.commit", "false");
  KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(properties, new StringDeserializer(),
      new StringDeserializer());
  kafkaConsumer.subscribe(Lists.newArrayList(patchTopic));
  return kafkaConsumer;
}
 
Example 23
Source Project: mewbase   Source File: KafkaEventSource.java    License: MIT License 5 votes vote down vote up
@Override
public CompletableFuture<Subscription> subscribeAll(String channelName, EventHandler eventHandler) {
    TopicPartition partition0 = new TopicPartition(channelName, partitionZeroOnly);
    KafkaConsumer<String, byte[]> kafkaConsumer = createAndAssignConsumer(partition0);
    kafkaConsumer.seekToBeginning(Arrays.asList(partition0));
    return CompletableFuture.completedFuture(createAndRegisterSubscription(kafkaConsumer,eventHandler));
}
 
Example 24
public KafkaMessageLogReceiverEndpointFactory(final MessageInterceptorRegistry interceptorRegistry,
                                              final KafkaConsumer<String, String> kafkaConsumer,
                                              final ExecutorService kinesisMessageLogExecutorService,
                                              final ApplicationEventPublisher eventPublisher) {
    this.interceptorRegistry = interceptorRegistry;
    this.kafkaConsumer = kafkaConsumer;
    this.executorService = kinesisMessageLogExecutorService;
    this.eventPublisher = eventPublisher;
}
 
Example 25
Source Project: springBoot-study   Source File: KafkaConsumerTest.java    License: Apache License 2.0 5 votes vote down vote up
private void init() {
		Properties props = new Properties();
		//kafka消费的的地址
		props.put("bootstrap.servers", "master:9092,slave1:9092,slave2:9092");
		//组名 不同组名可以重复消费
		props.put("group.id", GROUPID);
		//是否自动提交
		props.put("enable.auto.commit", "false");
		//超时时间
		props.put("session.timeout.ms", "30000");
		//一次最大拉取的条数
		props.put("max.poll.records", 10);
//		earliest当各分区下有已提交的offset时,从提交的offset开始消费;无提交的offset时,从头开始消费 
//		latest 
//		当各分区下有已提交的offset时,从提交的offset开始消费;无提交的offset时,消费新产生的该分区下的数据 
//		none 
//		topic各分区都存在已提交的offset时,从offset后开始消费;只要有一个分区不存在已提交的offset,则抛出异常
		props.put("auto.offset.reset", "earliest");
		//序列化
		props.put("key.deserializer", StringDeserializer.class.getName());
		props.put("value.deserializer", StringDeserializer.class.getName());
		this.consumer = new KafkaConsumer<String, String>(props);
		//订阅主题列表topic
		this.consumer.subscribe(Arrays.asList(topic));
		
		System.out.println("初始化!");
	}
 
Example 26
Source Project: jeesuite-libs   Source File: NewApiTopicConsumer.java    License: Apache License 2.0 5 votes vote down vote up
private void commitOffsets(ConsumerWorker worker) {
	
	KafkaConsumer<String, Serializable> consumer = worker.consumer;
	if(worker.isCommiting())return;
	worker.setCommiting(true);
	try {

		if(worker.uncommittedOffsetMap.isEmpty())return ;
		
		logger.debug("committing the offsets : {}", worker.uncommittedOffsetMap);
		consumer.commitAsync(worker.uncommittedOffsetMap, new OffsetCommitCallback() {
			@Override
			public void onComplete(Map<TopicPartition, OffsetAndMetadata> offsets, Exception exception) {
				//
				worker.setCommiting(false);
				if(exception == null){
					worker.resetUncommittedOffsetMap();
					logger.debug("committed the offsets : {}",offsets);
				}else{
					logger.error("committ the offsets error",exception);
				}
			}
		});
	} finally {
		
	}
}
 
Example 27
Source Project: SO   Source File: AConsumerHandler.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * constructor.<BR/>
 */
public AConsumerHandler(int id) {
    this.id = id;

    Properties properties = initProperties(getPropertiesMap());
    consumer = new KafkaConsumer<K, V>(properties);
}
 
Example 28
Source Project: kylin   Source File: KafkaClient.java    License: Apache License 2.0 5 votes vote down vote up
public static long getEarliestOffset(KafkaConsumer consumer, String topic, int partitionId) {

        TopicPartition topicPartition = new TopicPartition(topic, partitionId);
        consumer.assign(Arrays.asList(topicPartition));
        consumer.seekToBeginning(Arrays.asList(topicPartition));

        return consumer.position(topicPartition);
    }
 
Example 29
Source Project: kafka-webview   Source File: KafkaAdminFactory.java    License: MIT License 5 votes vote down vote up
/**
 * Create a new KafkaConsumer instance.
 * @param clusterConfig What cluster to connect to.
 * @param clientId What clientId to associate the connection with.
 * @return KafkaConsumer instance.
 */
public KafkaConsumer<String, String> createConsumer(final ClusterConfig clusterConfig, final String clientId) {
    // Create a map
    final Map<String, Object> config = configUtil.applyCommonSettings(clusterConfig, clientId);

    // Set required deserializer classes.
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

    // Create consumer
    return new KafkaConsumer<>(config);
}
 
Example 30
MetricLoader(KafkaConsumer<byte[], byte[]> consumer,
             SampleLoader sampleLoader,
             AtomicLong numLoadedSamples,
             AtomicLong numPartitionMetricSamples,
             AtomicLong numBrokerMetricSamples,
             AtomicLong totalSamples) {
  _consumer = consumer;
  _sampleLoader = sampleLoader;
  _numLoadedSamples = numLoadedSamples;
  _numPartitionMetricSamples = numPartitionMetricSamples;
  _numBrokerMetricSamples = numBrokerMetricSamples;
  _totalSamples = totalSamples;
}