kafka.javaapi.consumer.SimpleConsumer Java Examples
The following examples show how to use
kafka.javaapi.consumer.SimpleConsumer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaConsumer.java From jstorm with Apache License 2.0 | 6 votes |
public long getOffset(String topic, int partition, long startOffsetTime) { SimpleConsumer simpleConsumer = findLeaderConsumer(partition); if (simpleConsumer == null) { LOG.error("Error consumer is null get offset from partition:" + partition); return -1; } TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1)); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition); if (offsets.length > 0) { return offsets[0]; } else { return NO_OFFSET; } }
Example #2
Source File: KafkaRequest.java From HiveKa with Apache License 2.0 | 6 votes |
@Override public long getLastOffset(long time) { SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo( time, 1)); OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); if(endOffset.length == 0) { log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition); } this.latestOffset = endOffset[0]; return endOffset[0]; }
Example #3
Source File: KafkaRequest.java From HiveKa with Apache License 2.0 | 6 votes |
@Override public long getEarliestOffset() { if (this.earliestOffset == -2 && uri != null) { // TODO : Make the hardcoded paramters configurable SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000, 1024 * 1024, "hadoop-etl"); Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo( kafka.api.OffsetRequest.EarliestTime(), 1)); OffsetResponse response = consumer .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest .CurrentVersion(), "hadoop-etl")); long[] endOffset = response.offsets(topic, partition); consumer.close(); this.earliestOffset = endOffset[0]; return endOffset[0]; } else { return this.earliestOffset; } }
Example #4
Source File: NativeSimpleConsumer.java From spring-kafka-demo with Apache License 2.0 | 6 votes |
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition) ); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; }
Example #5
Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0 | 6 votes |
@Override public void close() throws IOException { int numOfConsumersNotClosed = 0; for (SimpleConsumer consumer : this.activeConsumers.values()) { if (consumer != null) { try { consumer.close(); } catch (Exception e) { LOG.warn(String.format("Failed to close Kafka Consumer %s:%d", consumer.host(), consumer.port())); numOfConsumersNotClosed++; } } } this.activeConsumers.clear(); if (numOfConsumersNotClosed > 0) { throw new IOException(numOfConsumersNotClosed + " consumer(s) failed to close."); } }
Example #6
Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0 | 6 votes |
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { LOG.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { LOG.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
Example #7
Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0 | 6 votes |
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) { log.info(String.format("Fetching topic metadata from broker %s", broker)); SimpleConsumer consumer = null; try { consumer = getSimpleConsumer(broker); for (int i = 0; i < this.fetchTopicRetries; i++) { try { return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata(); } catch (Exception e) { log.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e); try { Thread.sleep((long) ((i + Math.random()) * 1000)); } catch (InterruptedException e2) { log.warn("Caught InterruptedException: " + e2); } } } } finally { if (consumer != null) { consumer.close(); } } return null; }
Example #8
Source File: KafkaLeaderReader.java From arcusplatform with Apache License 2.0 | 6 votes |
KafkaLeaderReader( KafkaConsumerConfig config, KafkaConsumerFactory handlerFactory, Supplier<SimpleConsumer> supplier, List<TopicAndPartition> partitions ) { this.topic = config.getTopics(); this.clientId = config.getClientId(); this.scanFetchSize = config.getScanFetchSize(); this.fetchSize = config.getFetchSize(); this.emptyReadWait = config.getEmptyWaitTime(); this.emptyReadMax = config.getEmptyWaitMaxTime(); this.emptyExitTime = config.getEmptyExitTime(); this.handlers = new HashMap<>(); this.handlerFactory = handlerFactory; this.consumerFactory = supplier; this.partitions = ImmutableList.copyOf(partitions); this.offsets = new LinkedHashMap<>(); // we iterate this a lot }
Example #9
Source File: TridentKafkaEmitter.java From storm-kafka-0.8-plus with Apache License 2.0 | 6 votes |
/** * re-emit the batch described by the meta data provided * * @param attempt * @param collector * @param partition * @param meta */ private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) { LOG.info("re-emitting batch, attempt " + attempt); String instanceId = (String) meta.get("instanceId"); if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) { SimpleConsumer consumer = _connections.register(partition); long offset = (Long) meta.get("offset"); long nextOffset = (Long) meta.get("nextOffset"); ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset); for (MessageAndOffset msg : msgs) { if (offset == nextOffset) { break; } if (offset > nextOffset) { throw new RuntimeException("Error when re-emitting batch. overshot the end offset"); } emit(collector, msg.message()); offset = msg.nextOffset(); } } }
Example #10
Source File: KafkaComponent.java From metron with Apache License 2.0 | 6 votes |
public List<byte[]> readMessages(String topic) { SimpleConsumer consumer = new SimpleConsumer("localhost", 6667, 100000, 64 * 1024, "consumer"); FetchRequest req = new FetchRequestBuilder() .clientId("consumer") .addFetch(topic, 0, 0, 100000) .build(); FetchResponse fetchResponse = consumer.fetch(req); Iterator<MessageAndOffset> results = fetchResponse.messageSet(topic, 0).iterator(); List<byte[]> messages = new ArrayList<>(); while(results.hasNext()) { ByteBuffer payload = results.next().message().payload(); byte[] bytes = new byte[payload.limit()]; payload.get(bytes); messages.add(bytes); } consumer.close(); return messages; }
Example #11
Source File: KafkaSimpleConsumer.java From julongchain with Apache License 2.0 | 6 votes |
/** * 从保存consumer消费者offset偏移量的位置获取当前consumer对应的偏移量 * * @param consumer 消费者 * @param groupId Group Id * @param clientName client名称 * @param topic topic名称 * @param partitionID 分区id * @return */ public long getOffsetOfTopicAndPartition(SimpleConsumer consumer, String groupId, String clientName, String topic, int partitionID) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionID); List<TopicAndPartition> requestInfo = new ArrayList<TopicAndPartition>(); requestInfo.add(topicAndPartition); OffsetFetchRequest request = new OffsetFetchRequest(groupId, requestInfo, 0, clientName); OffsetFetchResponse response = consumer.fetchOffsets(request); // 获取返回值 Map<TopicAndPartition, OffsetMetadataAndError> returnOffsetMetadata = response.offsets(); // 处理返回值 if (returnOffsetMetadata != null && !returnOffsetMetadata.isEmpty()) { // 获取当前分区对应的偏移量信息 OffsetMetadataAndError offset = returnOffsetMetadata.get(topicAndPartition); if (offset.error().equals(ErrorMapping.NoError())) { // 没有异常,表示是正常的,获取偏移量 return offset.offset(); } else { // 当Consumer第一次连接的时候(zk中不在当前topic对应数据的时候),会产生UnknownTopicOrPartitionCode异常 System.out.println("Error fetching data Offset Data the Topic and Partition. Reason: " + offset.error()); } } // 所有异常情况直接返回0 return 0; }
Example #12
Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0 | 6 votes |
private long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) throws StageException { try { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest( requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(), response.errorCode(topic, partition)); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; } catch (Exception e) { LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e); throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e); } }
Example #13
Source File: KafkaMetadataUtil.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
/** * @param consumer * @param topic * @param partition * @param whichTime * @param clientName * @return 0 if consumer is null at this time */ public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) { if (consumer == null) { return 0; } TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition)); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; }
Example #14
Source File: SimpleKafkaMessageListener.java From micro-integrator with Apache License 2.0 | 6 votes |
public boolean run() throws Exception { if (init) { return init; } // find the meta data about the topic and partition we are interested in PartitionMetadata metadata = findLeader(seedBrokers, port, topic, partition); if (metadata == null) { throw new SynapseException("Can't find metadata for Topic and Partition. Exiting"); } if (metadata.leader() == null) { throw new SynapseException("Can't find Leader for Topic and Partition. Exiting"); } this.leadBroker = metadata.leader().host(); this.clientName = "Client_" + topic + "_" + partition; this.consumer = new SimpleConsumer(leadBroker, port, KAFKAConstants.BUFFER_SIZE, KAFKAConstants.SO_TIMEOUT, clientName); this.readOffset = getLastOffset(consumer, topic, partition, kafka.api.OffsetRequest.EarliestTime(), clientName); init = true; return init; }
Example #15
Source File: LegacyKafkaClient.java From secor with Apache License 2.0 | 6 votes |
@Override public int getNumPartitions(String topic) { SimpleConsumer consumer = null; try { consumer = createConsumer( mConfig.getKafkaSeedBrokerHost(), mConfig.getKafkaSeedBrokerPort(), "partitionLookup"); List<String> topics = new ArrayList<String>(); topics.add(topic); TopicMetadataRequest request = new TopicMetadataRequest(topics); TopicMetadataResponse response = consumer.send(request); if (response.topicsMetadata().size() != 1) { throw new RuntimeException("Expected one metadata for topic " + topic + " found " + response.topicsMetadata().size()); } TopicMetadata topicMetadata = response.topicsMetadata().get(0); return topicMetadata.partitionsMetadata().size(); } finally { if (consumer != null) { consumer.close(); } } }
Example #16
Source File: OffsetMonitor.java From uReplicator with Apache License 2.0 | 6 votes |
private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) { Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId()); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition, response.errorCode(topicAndPartition.topic(), topicAndPartition.partition())); return -1; } long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition()); return offsets[0]; }
Example #17
Source File: ZkConsumerCommand.java From jeesuite-libs with Apache License 2.0 | 6 votes |
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { System.out.println( "Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition)); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; }
Example #18
Source File: DemoLowLevelConsumer.java From KafkaExample with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception { final String topic = "topic1"; String clientID = "DemoLowLevelConsumer1"; SimpleConsumer simpleConsumer = new SimpleConsumer("kafka0", 9092, 100000, 64 * 1000000, clientID); FetchRequest req = new FetchRequestBuilder().clientId(clientID) .addFetch(topic, 0, 0L, 50).addFetch(topic, 1, 0L, 5000).addFetch(topic, 2, 0L, 1000000).build(); FetchResponse fetchResponse = simpleConsumer.fetch(req); ByteBufferMessageSet messageSet = (ByteBufferMessageSet) fetchResponse.messageSet(topic, 0); for (MessageAndOffset messageAndOffset : messageSet) { ByteBuffer payload = messageAndOffset.message().payload(); long offset = messageAndOffset.offset(); byte[] bytes = new byte[payload.limit()]; payload.get(bytes); System.out.println("Offset:" + offset + ", Payload:" + new String(bytes, "UTF-8")); } }
Example #19
Source File: KafkaLatestOffsetFetcher.java From eagle with Apache License 2.0 | 6 votes |
public Map<Integer, Long> fetch(String topic, int partitionCount) { Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount); Map<Integer, Long> ret = new HashMap<>(); for (int partition = 0; partition < partitionCount; partition++) { PartitionMetadata metadata = metadatas.get(partition); if (metadata == null || metadata.leader() == null) { ret.put(partition, -1L); //throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting"); } String leadBroker = metadata.leader().host(); String clientName = "Client_" + topic + "_" + partition; SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName); long latestOffset = getLatestOffset(consumer, topic, partition, clientName); if (consumer != null) consumer.close(); ret.put(partition, latestOffset); } return ret; }
Example #20
Source File: MetricSystemTest.java From eagle with Apache License 2.0 | 6 votes |
@Test public void testMerticSystemWithKafkaSink() throws IOException { JVMMetricSource jvmMetricSource = mockMetricRegistry(); //setup kafka KafkaEmbedded kafkaEmbedded = new KafkaEmbedded(); makeSureTopic(kafkaEmbedded.getZkConnectionString()); //setup metric system File file = genKafkaSinkConfig(kafkaEmbedded.getBrokerConnectionString()); Config config = ConfigFactory.parseFile(file); MetricSystem system = MetricSystem.load(config); system.register(jvmMetricSource); system.start(); system.report(); SimpleConsumer consumer = assertMsgFromKafka(kafkaEmbedded); system.stop(); consumer.close(); kafkaEmbedded.shutdown(); }
Example #21
Source File: ZkConsumerCommand.java From azeroth with Apache License 2.0 | 6 votes |
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition)); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; }
Example #22
Source File: KafkaLatestOffsetFetcher.java From eagle with Apache License 2.0 | 6 votes |
public Map<Integer, Long> fetch(String topic, int partitionCount) { Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount); Map<Integer, Long> ret = new HashMap<>(); for (int partition = 0; partition < partitionCount; partition++) { PartitionMetadata metadata = metadatas.get(partition); if (metadata == null || metadata.leader() == null) { ret.put(partition, -1L); //throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting"); } String leadBroker = metadata.leader().host(); String clientName = "Client_" + topic + "_" + partition; SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName); long latestOffset = getLatestOffset(consumer, topic, partition, clientName); if (consumer != null) { consumer.close(); } ret.put(partition, latestOffset); } return ret; }
Example #23
Source File: SimpleConsumerExample.java From hermes with Apache License 2.0 | 6 votes |
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) { TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition); Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>(); requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1)); kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName); OffsetResponse response = consumer.getOffsetsBefore(request); if (response.hasError()) { System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition)); return 0; } long[] offsets = response.offsets(topic, partition); return offsets[0]; }
Example #24
Source File: KafkaClientTest.java From elasticsearch-river-kafka with Apache License 2.0 | 6 votes |
@Override protected void setUp() throws Exception { super.setUp(); mockCurator = createMock(CuratorFramework.class); mockConsumer = createMock(SimpleConsumer.class); final CuratorFramework cur = mockCurator; final SimpleConsumer con = mockConsumer; client = new KafkaClient("zookeeper", "broker", 9092){ void connect(String zk, String broker, int port) { this.curator = cur; this.consumer = con; }; }; }
Example #25
Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0 | 5 votes |
private SimpleConsumer getSimpleConsumer(String broker) { if (this.activeConsumers.containsKey(broker)) { return this.activeConsumers.get(broker); } SimpleConsumer consumer = this.createSimpleConsumer(broker); this.activeConsumers.putIfAbsent(broker, consumer); return consumer; }
Example #26
Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0 | 5 votes |
private SimpleConsumer getSimpleConsumer(String broker) { if (this.activeConsumers.containsKey(broker)) { return this.activeConsumers.get(broker); } SimpleConsumer consumer = this.createSimpleConsumer(broker); this.activeConsumers.putIfAbsent(broker, consumer); return consumer; }
Example #27
Source File: KafkaMessageReceiverImpl.java From message-queue-client-framework with Apache License 2.0 | 5 votes |
/** * Check the consumer and create consumer. * * @param a_topic topic name * @param a_partition partition number * @return boolean */ private boolean checkConsumer(String a_topic, int a_partition) { if (consumer.get() == null) { if (metadata == null) { replicaBrokers.clear(); String brokerStr = getBrokerStr(a_topic); String[] brokers = brokerStr.split(","); for (String broker : brokers) { String[] hostport = broker.split(":"); replicaBrokers.put(hostport[0], Integer.valueOf(hostport[1])); } metadata = findLeader(replicaBrokers, a_topic, a_partition); } if (metadata == null) { logger.error("Can't find metadata for Topic and Partition. Exiting"); return false; } if (metadata.leader() == null) { logger.error("Can't find Leader for Topic and Partition. Exiting"); return false; } String leadHost = metadata.leader().host(); Integer leadPort = metadata.leader().port(); String clientName = pool.getClientId(); consumer.compareAndSet(null, new SimpleConsumer(leadHost, leadPort, KafkaConstants.SO_TIMEOUT, KafkaConstants.BUFFER_SIZE, clientName)); } return true; }
Example #28
Source File: ZkConsumerCommand.java From jeesuite-libs with Apache License 2.0 | 5 votes |
/** * 获取指定主题及分区logsize * @param stat */ public void getTopicPartitionLogSize(TopicPartitionInfo stat){ BrokerEndPoint leader = findLeader(stat.getTopic(), stat.getPartition()).leader(); SimpleConsumer consumer = getConsumerClient(leader.host(), leader.port()); try { long logsize = getLastOffset(consumer,stat.getTopic(), stat.getPartition(), kafka.api.OffsetRequest.LatestTime()); stat.setLogSize(logsize); } finally { consumer.close(); } }
Example #29
Source File: ZkConsumerCommand.java From jeesuite-libs with Apache License 2.0 | 5 votes |
private SimpleConsumer getConsumerClient(String kafkaServer){ if(consumers.containsKey(kafkaServer))return consumers.get(kafkaServer); String host = kafkaServer.split(":")[0]; int port = Integer.parseInt(kafkaServer.split(":")[1]); SimpleConsumer consumer = new SimpleConsumer(host, port, 100000, 64 * 1024, CLIENT_ID); consumers.put(kafkaServer, consumer); return consumer; }
Example #30
Source File: KafkaUtilsTest.java From storm-kafka-0.8-plus with Apache License 2.0 | 5 votes |
@Test(expected = FailedFetchException.class) public void brokerIsDown() throws Exception { int port = broker.getPort(); broker.shutdown(); SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", port, 100, 1024, "testClient"); try { KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), OffsetRequest.LatestTime()); } finally { simpleConsumer.close(); } }