kafka.javaapi.consumer.SimpleConsumer Java Examples

The following examples show how to use kafka.javaapi.consumer.SimpleConsumer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaConsumer.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public long getOffset(String topic, int partition, long startOffsetTime) {
    SimpleConsumer simpleConsumer = findLeaderConsumer(partition);

    if (simpleConsumer == null) {
        LOG.error("Error consumer is null get offset from partition:" + partition);
        return -1;
    }

    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());

    long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}
 
Example #2
Source File: KafkaRequest.java    From HiveKa with Apache License 2.0 6 votes vote down vote up
@Override
public long getLastOffset(long time) {
  SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
      1024 * 1024, "hadoop-etl");
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
      time, 1));
  OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo,
      kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl"));
  long[] endOffset = response.offsets(topic, partition);
  consumer.close();
  if(endOffset.length == 0)
  {
    log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition);
  }
  this.latestOffset = endOffset[0];
  return endOffset[0];
}
 
Example #3
Source File: KafkaRequest.java    From HiveKa with Apache License 2.0 6 votes vote down vote up
@Override
public long getEarliestOffset() {
  if (this.earliestOffset == -2 && uri != null) {
    // TODO : Make the hardcoded paramters configurable
    SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
        1024 * 1024, "hadoop-etl");
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
        kafka.api.OffsetRequest.EarliestTime(), 1));
    OffsetResponse response = consumer
        .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest
            .CurrentVersion(), "hadoop-etl"));
    long[] endOffset = response.offsets(topic, partition);
    consumer.close();
    this.earliestOffset = endOffset[0];
    return endOffset[0];
  } else {
    return this.earliestOffset;
  }
}
 
Example #4
Source File: NativeSimpleConsumer.java    From spring-kafka-demo with Apache License 2.0 6 votes vote down vote up
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
 
    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition) );
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 
Example #5
Source File: KafkaWrapper.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws IOException {
  int numOfConsumersNotClosed = 0;

  for (SimpleConsumer consumer : this.activeConsumers.values()) {
    if (consumer != null) {
      try {
        consumer.close();
      } catch (Exception e) {
        LOG.warn(String.format("Failed to close Kafka Consumer %s:%d", consumer.host(), consumer.port()));
        numOfConsumersNotClosed++;
      }
    }
  }
  this.activeConsumers.clear();
  if (numOfConsumersNotClosed > 0) {
    throw new IOException(numOfConsumersNotClosed + " consumer(s) failed to close.");
  }
}
 
Example #6
Source File: KafkaWrapper.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  LOG.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < this.fetchTopicRetries; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        LOG.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 
Example #7
Source File: Kafka08ConsumerClient.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private List<TopicMetadata> fetchTopicMetadataFromBroker(String broker, String... selectedTopics) {
  log.info(String.format("Fetching topic metadata from broker %s", broker));
  SimpleConsumer consumer = null;
  try {
    consumer = getSimpleConsumer(broker);
    for (int i = 0; i < this.fetchTopicRetries; i++) {
      try {
        return consumer.send(new TopicMetadataRequest(Arrays.asList(selectedTopics))).topicsMetadata();
      } catch (Exception e) {
        log.warn(String.format("Fetching topic metadata from broker %s has failed %d times.", broker, i + 1), e);
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          log.warn("Caught InterruptedException: " + e2);
        }
      }
    }
  } finally {
    if (consumer != null) {
      consumer.close();
    }
  }
  return null;
}
 
Example #8
Source File: KafkaLeaderReader.java    From arcusplatform with Apache License 2.0 6 votes vote down vote up
KafkaLeaderReader(
      KafkaConsumerConfig config,
      KafkaConsumerFactory handlerFactory,
      Supplier<SimpleConsumer> supplier,
      List<TopicAndPartition> partitions
) {
   this.topic = config.getTopics();
   this.clientId = config.getClientId();
   this.scanFetchSize = config.getScanFetchSize();
   this.fetchSize = config.getFetchSize();
   this.emptyReadWait = config.getEmptyWaitTime();
   this.emptyReadMax = config.getEmptyWaitMaxTime();
   this.emptyExitTime = config.getEmptyExitTime();

   this.handlers = new HashMap<>();
   this.handlerFactory = handlerFactory;
   this.consumerFactory = supplier;
   this.partitions = ImmutableList.copyOf(partitions);

   this.offsets = new LinkedHashMap<>(); // we iterate this a lot
}
 
Example #9
Source File: TridentKafkaEmitter.java    From storm-kafka-0.8-plus with Apache License 2.0 6 votes vote down vote up
/**
 * re-emit the batch described by the meta data provided
 *
 * @param attempt
 * @param collector
 * @param partition
 * @param meta
 */
private void reEmitPartitionBatch(TransactionAttempt attempt, TridentCollector collector, Partition partition, Map meta) {
    LOG.info("re-emitting batch, attempt " + attempt);
    String instanceId = (String) meta.get("instanceId");
    if (!_config.forceFromStart || instanceId.equals(_topologyInstanceId)) {
        SimpleConsumer consumer = _connections.register(partition);
        long offset = (Long) meta.get("offset");
        long nextOffset = (Long) meta.get("nextOffset");
        ByteBufferMessageSet msgs = fetchMessages(consumer, partition, offset);
        for (MessageAndOffset msg : msgs) {
            if (offset == nextOffset) {
                break;
            }
            if (offset > nextOffset) {
                throw new RuntimeException("Error when re-emitting batch. overshot the end offset");
            }
            emit(collector, msg.message());
            offset = msg.nextOffset();
        }
    }
}
 
Example #10
Source File: KafkaComponent.java    From metron with Apache License 2.0 6 votes vote down vote up
public List<byte[]> readMessages(String topic) {
  SimpleConsumer consumer = new SimpleConsumer("localhost", 6667, 100000, 64 * 1024, "consumer");
  FetchRequest req = new FetchRequestBuilder()
          .clientId("consumer")
          .addFetch(topic, 0, 0, 100000)
          .build();
  FetchResponse fetchResponse = consumer.fetch(req);
  Iterator<MessageAndOffset> results = fetchResponse.messageSet(topic, 0).iterator();
  List<byte[]> messages = new ArrayList<>();
  while(results.hasNext()) {
    ByteBuffer payload = results.next().message().payload();
    byte[] bytes = new byte[payload.limit()];
    payload.get(bytes);
    messages.add(bytes);
  }
  consumer.close();
  return messages;
}
 
Example #11
Source File: KafkaSimpleConsumer.java    From julongchain with Apache License 2.0 6 votes vote down vote up
/**
 * 从保存consumer消费者offset偏移量的位置获取当前consumer对应的偏移量
 *
 * @param consumer    消费者
 * @param groupId     Group Id
 * @param clientName  client名称
 * @param topic       topic名称
 * @param partitionID 分区id
 * @return
 */
public long getOffsetOfTopicAndPartition(SimpleConsumer consumer, String groupId, String clientName, String topic, int partitionID) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionID);
    List<TopicAndPartition> requestInfo = new ArrayList<TopicAndPartition>();
    requestInfo.add(topicAndPartition);
    OffsetFetchRequest request = new OffsetFetchRequest(groupId, requestInfo, 0, clientName);
    OffsetFetchResponse response = consumer.fetchOffsets(request);

    // 获取返回值
    Map<TopicAndPartition, OffsetMetadataAndError> returnOffsetMetadata = response.offsets();
    // 处理返回值
    if (returnOffsetMetadata != null && !returnOffsetMetadata.isEmpty()) {
        // 获取当前分区对应的偏移量信息
        OffsetMetadataAndError offset = returnOffsetMetadata.get(topicAndPartition);
        if (offset.error().equals(ErrorMapping.NoError())) {
            // 没有异常,表示是正常的,获取偏移量
            return offset.offset();
        } else {
            // 当Consumer第一次连接的时候(zk中不在当前topic对应数据的时候),会产生UnknownTopicOrPartitionCode异常
            System.out.println("Error fetching data Offset Data the Topic and Partition. Reason: " + offset.error());
        }
    }

    // 所有异常情况直接返回0
    return 0;
}
 
Example #12
Source File: KafkaLowLevelConsumer08.java    From datacollector with Apache License 2.0 6 votes vote down vote up
private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) throws StageException {
  try {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
        response.errorCode(topic, partition));
      return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  } catch (Exception e) {
    LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
  }
}
 
Example #13
Source File: KafkaMetadataUtil.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
/**
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return 0 if consumer is null at this time
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName)
{
  if (consumer == null) {
    return 0;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
    return 0;
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}
 
Example #14
Source File: SimpleKafkaMessageListener.java    From micro-integrator with Apache License 2.0 6 votes vote down vote up
public boolean run() throws Exception {

        if (init) {
            return init;
        }
        // find the meta data about the topic and partition we are interested in
        PartitionMetadata metadata = findLeader(seedBrokers, port, topic, partition);
        if (metadata == null) {
            throw new SynapseException("Can't find metadata for Topic and Partition. Exiting");
        }
        if (metadata.leader() == null) {
            throw new SynapseException("Can't find Leader for Topic and Partition. Exiting");
        }
        this.leadBroker = metadata.leader().host();
        this.clientName = "Client_" + topic + "_" + partition;

        this.consumer = new SimpleConsumer(leadBroker, port, KAFKAConstants.BUFFER_SIZE, KAFKAConstants.SO_TIMEOUT,
                                           clientName);
        this.readOffset = getLastOffset(consumer, topic, partition, kafka.api.OffsetRequest.EarliestTime(), clientName);
        init = true;

        return init;
    }
 
Example #15
Source File: LegacyKafkaClient.java    From secor with Apache License 2.0 6 votes vote down vote up
@Override
public int getNumPartitions(String topic) {
    SimpleConsumer consumer = null;
    try {
        consumer = createConsumer(
            mConfig.getKafkaSeedBrokerHost(),
            mConfig.getKafkaSeedBrokerPort(),
            "partitionLookup");
        List<String> topics = new ArrayList<String>();
        topics.add(topic);
        TopicMetadataRequest request = new TopicMetadataRequest(topics);
        TopicMetadataResponse response = consumer.send(request);
        if (response.topicsMetadata().size() != 1) {
            throw new RuntimeException("Expected one metadata for topic " + topic + " found " +
                response.topicsMetadata().size());
        }
        TopicMetadata topicMetadata = response.topicsMetadata().get(0);
        return topicMetadata.partitionsMetadata().size();
    } finally {
        if (consumer != null) {
            consumer.close();
        }
    }
}
 
Example #16
Source File: OffsetMonitor.java    From uReplicator with Apache License 2.0 6 votes vote down vote up
private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition,
      new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
      kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
        response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
    return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
}
 
Example #17
Source File: ZkConsumerCommand.java    From jeesuite-libs with Apache License 2.0 6 votes vote down vote up
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println(
				"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 
Example #18
Source File: DemoLowLevelConsumer.java    From KafkaExample with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final String topic = "topic1";
	String clientID = "DemoLowLevelConsumer1";
	SimpleConsumer simpleConsumer = new SimpleConsumer("kafka0", 9092, 100000, 64 * 1000000, clientID);
	FetchRequest req = new FetchRequestBuilder().clientId(clientID)
			.addFetch(topic, 0, 0L, 50).addFetch(topic, 1, 0L, 5000).addFetch(topic, 2, 0L, 1000000).build();
	FetchResponse fetchResponse = simpleConsumer.fetch(req);
	ByteBufferMessageSet messageSet = (ByteBufferMessageSet) fetchResponse.messageSet(topic, 0);
	for (MessageAndOffset messageAndOffset : messageSet) {
		ByteBuffer payload = messageAndOffset.message().payload();
		long offset = messageAndOffset.offset();
		byte[] bytes = new byte[payload.limit()];
		payload.get(bytes);
		System.out.println("Offset:" + offset + ", Payload:" + new String(bytes, "UTF-8"));
	}
}
 
Example #19
Source File: KafkaLatestOffsetFetcher.java    From eagle with Apache License 2.0 6 votes vote down vote up
public Map<Integer, Long> fetch(String topic, int partitionCount) {
    Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount);
    Map<Integer, Long> ret = new HashMap<>();
    for (int partition = 0; partition < partitionCount; partition++) {
        PartitionMetadata metadata = metadatas.get(partition);
        if (metadata == null || metadata.leader() == null) {
            ret.put(partition, -1L);
            //throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting");
        }
        String leadBroker = metadata.leader().host();
        String clientName = "Client_" + topic + "_" + partition;
        SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);
        long latestOffset = getLatestOffset(consumer, topic, partition, clientName);
        if (consumer != null) consumer.close();
        ret.put(partition, latestOffset);
    }
    return ret;
}
 
Example #20
Source File: MetricSystemTest.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Test
public void testMerticSystemWithKafkaSink() throws IOException {

    JVMMetricSource jvmMetricSource = mockMetricRegistry();
    //setup kafka
    KafkaEmbedded kafkaEmbedded = new KafkaEmbedded();
    makeSureTopic(kafkaEmbedded.getZkConnectionString());
    //setup metric system
    File file = genKafkaSinkConfig(kafkaEmbedded.getBrokerConnectionString());
    Config config = ConfigFactory.parseFile(file);
    MetricSystem system = MetricSystem.load(config);
    system.register(jvmMetricSource);
    system.start();
    system.report();

    SimpleConsumer consumer = assertMsgFromKafka(kafkaEmbedded);
    system.stop();
    consumer.close();
    kafkaEmbedded.shutdown();
}
 
Example #21
Source File: ZkConsumerCommand.java    From azeroth with Apache License 2.0 6 votes vote down vote up
private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}
 
Example #22
Source File: KafkaLatestOffsetFetcher.java    From eagle with Apache License 2.0 6 votes vote down vote up
public Map<Integer, Long> fetch(String topic, int partitionCount) {
    Map<Integer, PartitionMetadata> metadatas = fetchPartitionMetadata(brokerList, port, topic, partitionCount);
    Map<Integer, Long> ret = new HashMap<>();
    for (int partition = 0; partition < partitionCount; partition++) {
        PartitionMetadata metadata = metadatas.get(partition);
        if (metadata == null || metadata.leader() == null) {
            ret.put(partition, -1L);
            //throw new RuntimeException("Can't find Leader for Topic and Partition. Exiting");
        }
        String leadBroker = metadata.leader().host();
        String clientName = "Client_" + topic + "_" + partition;
        SimpleConsumer consumer = new SimpleConsumer(leadBroker, port, 100000, 64 * 1024, clientName);
        long latestOffset = getLatestOffset(consumer, topic, partition, clientName);
        if (consumer != null) {
            consumer.close();
        }
        ret.put(partition, latestOffset);
    }
    return ret;
}
 
Example #23
Source File: SimpleConsumerExample.java    From hermes with Apache License 2.0 6 votes vote down vote up
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
      String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
	      kafka.api.OffsetRequest.CurrentVersion(), clientName);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println("Error fetching data Offset Data the Broker. Reason: "
		      + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}
 
Example #24
Source File: KafkaClientTest.java    From elasticsearch-river-kafka with Apache License 2.0 6 votes vote down vote up
@Override
protected void setUp() throws Exception {
	super.setUp();
	mockCurator = createMock(CuratorFramework.class);
	mockConsumer = createMock(SimpleConsumer.class);
	
	final CuratorFramework cur = mockCurator;
	final SimpleConsumer con = mockConsumer;
	client = new KafkaClient("zookeeper", "broker", 9092){
		void connect(String zk, String broker, int port) 
		{
			this.curator = cur;
			this.consumer = con;
		};
	};
}
 
Example #25
Source File: Kafka08ConsumerClient.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private SimpleConsumer getSimpleConsumer(String broker) {
  if (this.activeConsumers.containsKey(broker)) {
    return this.activeConsumers.get(broker);
  }
  SimpleConsumer consumer = this.createSimpleConsumer(broker);
  this.activeConsumers.putIfAbsent(broker, consumer);
  return consumer;
}
 
Example #26
Source File: KafkaWrapper.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private SimpleConsumer getSimpleConsumer(String broker) {
  if (this.activeConsumers.containsKey(broker)) {
    return this.activeConsumers.get(broker);
  }
  SimpleConsumer consumer = this.createSimpleConsumer(broker);
  this.activeConsumers.putIfAbsent(broker, consumer);
  return consumer;
}
 
Example #27
Source File: KafkaMessageReceiverImpl.java    From message-queue-client-framework with Apache License 2.0 5 votes vote down vote up
/**
 * Check the consumer and create consumer.
 *
 * @param a_topic     topic name
 * @param a_partition partition number
 * @return boolean
 */
private boolean checkConsumer(String a_topic, int a_partition) {

    if (consumer.get() == null) {

        if (metadata == null) {

            replicaBrokers.clear();
            String brokerStr = getBrokerStr(a_topic);
            String[] brokers = brokerStr.split(",");
            for (String broker : brokers) {
                String[] hostport = broker.split(":");
                replicaBrokers.put(hostport[0],
                        Integer.valueOf(hostport[1]));
            }
            metadata = findLeader(replicaBrokers, a_topic, a_partition);
        }

        if (metadata == null) {
            logger.error("Can't find metadata for Topic and Partition. Exiting");
            return false;
        }
        if (metadata.leader() == null) {
            logger.error("Can't find Leader for Topic and Partition. Exiting");
            return false;
        }
        String leadHost = metadata.leader().host();
        Integer leadPort = metadata.leader().port();
        String clientName = pool.getClientId();

        consumer.compareAndSet(null, new SimpleConsumer(leadHost, leadPort,
                KafkaConstants.SO_TIMEOUT, KafkaConstants.BUFFER_SIZE,
                clientName));
    }

    return true;
}
 
Example #28
Source File: ZkConsumerCommand.java    From jeesuite-libs with Apache License 2.0 5 votes vote down vote up
/**
   * 获取指定主题及分区logsize
   * @param stat
   */
  public void getTopicPartitionLogSize(TopicPartitionInfo stat){
  	BrokerEndPoint leader = findLeader(stat.getTopic(), stat.getPartition()).leader();
  	SimpleConsumer consumer = getConsumerClient(leader.host(), leader.port());	
  	
  	try {			
  		long logsize = getLastOffset(consumer,stat.getTopic(), stat.getPartition(), kafka.api.OffsetRequest.LatestTime());
  		stat.setLogSize(logsize);
} finally {
	consumer.close();
}
  }
 
Example #29
Source File: ZkConsumerCommand.java    From jeesuite-libs with Apache License 2.0 5 votes vote down vote up
private SimpleConsumer getConsumerClient(String kafkaServer){
  	if(consumers.containsKey(kafkaServer))return consumers.get(kafkaServer);
  	String host = kafkaServer.split(":")[0];
int port = Integer.parseInt(kafkaServer.split(":")[1]);

SimpleConsumer consumer = new SimpleConsumer(host, port, 100000, 64 * 1024, CLIENT_ID);
consumers.put(kafkaServer, consumer);
return consumer;
  }
 
Example #30
Source File: KafkaUtilsTest.java    From storm-kafka-0.8-plus with Apache License 2.0 5 votes vote down vote up
@Test(expected = FailedFetchException.class)
public void brokerIsDown() throws Exception {
    int port = broker.getPort();
    broker.shutdown();
    SimpleConsumer simpleConsumer = new SimpleConsumer("localhost", port, 100, 1024, "testClient");
    try {
        KafkaUtils.fetchMessages(config, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), OffsetRequest.LatestTime());
    } finally {
        simpleConsumer.close();
    }
}