kafka.api.PartitionOffsetRequestInfo Java Examples

The following examples show how to use kafka.api.PartitionOffsetRequestInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ZkConsumerCommand.java From jeesuite-libs with Apache License 2.0

6 votes

private static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
			kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println(
				"Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}

Example #2

Source File: KafkaLowLevelConsumer09.java From datacollector with Apache License 2.0

6 votes

private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) throws StageException {
  try {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
        response.errorCode(topic, partition));
      return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  } catch (Exception e) {
    LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
  }
}

Example #3

Source File: KafkaLowLevelConsumer08.java From datacollector with Apache License 2.0

6 votes

private long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) throws StageException {
  try {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
      LOG.error(KafkaErrors.KAFKA_22.getMessage(), consumer.host() + ":" + consumer.port(),
        response.errorCode(topic, partition));
      return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
  } catch (Exception e) {
    LOG.error(KafkaErrors.KAFKA_30.getMessage(), e.toString(), e);
    throw new StageException(KafkaErrors.KAFKA_30, e.toString(), e);
  }
}

Example #4

Source File: KafkaMetadataUtil.java From attic-apex-malhar with Apache License 2.0

6 votes

/**
 * @param consumer
 * @param topic
 * @param partition
 * @param whichTime
 * @param clientName
 * @return 0 if consumer is null at this time
 */
public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName)
{
  if (consumer == null) {
    return 0;
  }
  TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
  OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.error("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition));
    return 0;
  }
  long[] offsets = response.offsets(topic, partition);
  return offsets[0];
}

Example #5

Source File: KafkaRequest.java From HiveKa with Apache License 2.0

6 votes

@Override
public long getEarliestOffset() {
  if (this.earliestOffset == -2 && uri != null) {
    // TODO : Make the hardcoded paramters configurable
    SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
        1024 * 1024, "hadoop-etl");
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
        kafka.api.OffsetRequest.EarliestTime(), 1));
    OffsetResponse response = consumer
        .getOffsetsBefore(new OffsetRequest(offsetInfo, kafka.api.OffsetRequest
            .CurrentVersion(), "hadoop-etl"));
    long[] endOffset = response.offsets(topic, partition);
    consumer.close();
    this.earliestOffset = endOffset[0];
    return endOffset[0];
  } else {
    return this.earliestOffset;
  }
}

Example #6

Source File: KafkaRequest.java From HiveKa with Apache License 2.0

6 votes

@Override
public long getLastOffset(long time) {
  SimpleConsumer consumer = new SimpleConsumer(uri.getHost(), uri.getPort(), 60000,
      1024 * 1024, "hadoop-etl");
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
  offsetInfo.put(new TopicAndPartition(topic, partition), new PartitionOffsetRequestInfo(
      time, 1));
  OffsetResponse response = consumer.getOffsetsBefore(new OffsetRequest(offsetInfo,
      kafka.api.OffsetRequest.CurrentVersion(),"hadoop-etl"));
  long[] endOffset = response.offsets(topic, partition);
  consumer.close();
  if(endOffset.length == 0)
  {
    log.info("The exception is thrown because the latest offset retunred zero for topic : " + topic + " and partition " + partition);
  }
  this.latestOffset = endOffset[0];
  return endOffset[0];
}

Example #7

Source File: KafkaMessageReceiverImpl.java From message-queue-client-framework with Apache License 2.0

6 votes

@Override
public synchronized long getEarliestOffset(String topic, int partition) {

    if (checkConsumer(topic, partition)) {

        TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
                partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.EarliestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                pool.getClientId());
        OffsetResponse response = consumer.get().getOffsetsBefore(request);

        if (response.hasError()) {
            logger.error("Error fetching data Offset Data the Broker. Reason: "
                    + response.errorCode(topic, partition));
            return 0;
        }
        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }

    return -1;
}

Example #8

Source File: KafkaMessageReceiverImpl.java From message-queue-client-framework with Apache License 2.0

6 votes

@Override
public synchronized long getLatestOffset(String topic, int partition) {

    if (checkConsumer(topic, partition)) {

        TopicAndPartition topicAndPartition = new TopicAndPartition(topic,
                partition);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
                kafka.api.OffsetRequest.LatestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(),
                pool.getClientId());
        OffsetResponse response = consumer.get().getOffsetsBefore(request);

        if (response.hasError()) {
            logger.error("Error fetching data Offset Data the Broker. Reason: "
                    + response.errorCode(topic, partition));
            return 0;
        }
        long[] offsets = response.offsets(topic, partition);
        return offsets[0];
    }
    return -1;
}

Example #9

Source File: NativeSimpleConsumer.java From spring-kafka-demo with Apache License 2.0

6 votes

public static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                 long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
 
    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: " + response.errorCode(topic, partition) );
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}

Example #10

Source File: OffsetMonitor.java From uReplicator with Apache License 2.0

6 votes

private long getLatestOffset(SimpleConsumer consumer, TopicAndPartition topicAndPartition) {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
  requestInfo.put(topicAndPartition,
      new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
      kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());
  OffsetResponse response = consumer.getOffsetsBefore(request);

  if (response.hasError()) {
    logger.warn("Failed to fetch offset for {} due to {}", topicAndPartition,
        response.errorCode(topicAndPartition.topic(), topicAndPartition.partition()));
    return -1;
  }

  long[] offsets = response.offsets(topicAndPartition.topic(), topicAndPartition.partition());
  return offsets[0];
}

Example #11

Source File: ZkConsumerCommand.java From azeroth with Apache License 2.0

6 votes

private static long getLastOffset(SimpleConsumer consumer, String topic, int partition,
                                  long whichTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
        kafka.api.OffsetRequest.CurrentVersion(), CLIENT_ID);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        System.out.println("Error fetching data Offset Data the Broker. Reason: "
                           + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}

Example #12

Source File: KafkaConsumer.java From jstorm with Apache License 2.0

6 votes

public long getOffset(String topic, int partition, long startOffsetTime) {
    SimpleConsumer simpleConsumer = findLeaderConsumer(partition);

    if (simpleConsumer == null) {
        LOG.error("Error consumer is null get offset from partition:" + partition);
        return -1;
    }

    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId());

    long[] offsets = simpleConsumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}

Example #13

Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0

6 votes

private long getEarliestOffsetFromKafka(String topic, int partition, long startTime) {
  LOG.info("getEarliestOffsetFromKafka.");
  TopicAndPartition tp = new TopicAndPartition(topic, partition);

  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();

  requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));

  OffsetRequest req = new OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());

  OffsetResponse resp = consumer.getOffsetsBefore(req);

  if (resp.hasError()) {
    LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
    return 0;
  }
  LOG.info("Earliest offset " + resp.offsets(topic, partition)[0]);
  return resp.offsets(topic, partition)[0];
}

Example #14

Source File: LegacyKafkaClient.java From secor with Apache License 2.0

6 votes

private long findLastOffset(TopicPartition topicPartition, SimpleConsumer consumer) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topicPartition.getTopic(),
            topicPartition.getPartition());
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
            new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
            kafka.api.OffsetRequest.LatestTime(), 1));
    final String clientName = getClientName(topicPartition);
    OffsetRequest request = new OffsetRequest(requestInfo,
                                              kafka.api.OffsetRequest.CurrentVersion(),
                                              clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);

    if (response.hasError()) {
        throw new RuntimeException("Error fetching offset data. Reason: " +
                response.errorCode(topicPartition.getTopic(), topicPartition.getPartition()));
    }
    long[] offsets = response.offsets(topicPartition.getTopic(),
            topicPartition.getPartition());
    return offsets[0] - 1;
}

Example #15

Source File: KafkaUtils.java From kafka-monitor with Apache License 2.0

6 votes

/**
 * 请求取topic偏移
 *
 * @param broker
 * @param topic
 * @param brokerPartitions
 * @return
 */
public static OffsetResponse sendOffsetRequest(Broker broker, Topic topic,
                                               List<Partition> brokerPartitions, long time) {

    PartitionOffsetRequestInfo requestInfo = new PartitionOffsetRequestInfo(time, 1);

    final OffsetRequest offsetRequest = new OffsetRequest(
            brokerPartitions.stream()
                    .collect(Collectors.toMap(
                            partition -> new TopicAndPartition(topic.getName(), partition.getId()),
                            partition -> requestInfo)), (short) 0, "kafkaMonitor");

    logger.debug("Sending offset request: {}", offsetRequest);
    if (broker != null) {
        BlockingChannel channel = getChannel(broker);
        channel.send(offsetRequest.underlying());
        final kafka.api.OffsetResponse underlyingResponse = kafka.api.OffsetResponse.readFrom(channel.receive().payload());
        channel.disconnect();
        return new OffsetResponse(underlyingResponse);
    } else {
        return null;
    }

}

Example #16

Source File: SimpleConsumerExample.java From hermes with Apache License 2.0

6 votes

public static long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime,
      String clientName) {
	TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
	requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
	kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(requestInfo,
	      kafka.api.OffsetRequest.CurrentVersion(), clientName);
	OffsetResponse response = consumer.getOffsetsBefore(request);

	if (response.hasError()) {
		System.out.println("Error fetching data Offset Data the Broker. Reason: "
		      + response.errorCode(topic, partition));
		return 0;
	}
	long[] offsets = response.offsets(topic, partition);
	return offsets[0];
}

Example #17

Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0

5 votes

@Override
protected long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
      Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
          new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
  return getOffset(partition, offsetRequestInfo);
}

Example #18

Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0

5 votes

@Override
protected long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
      Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
          new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  return getOffset(partition, offsetRequestInfo);
}

Example #19

Source File: KafkaWrapper.java From incubator-gobblin with Apache License 2.0

5 votes

private long getOffset(KafkaPartition partition,
    Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
    throws KafkaOffsetRetrievalFailureException {
  SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
  for (int i = 0; i < this.fetchOffsetRetries; i++) {
    try {
      OffsetResponse offsetResponse = consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo,
          kafka.api.OffsetRequest.CurrentVersion(), this.clientName));
      if (offsetResponse.hasError()) {
        throw new RuntimeException(
            "offsetReponse has error: " + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
      }
      return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
    } catch (Exception e) {
      LOG.warn(
          String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1, e));
      if (i < this.fetchOffsetRetries - 1) {
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          LOG.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
        }
      }
    }
  }
  throw new KafkaOffsetRetrievalFailureException(
      String.format("Fetching offset for partition %s has failed.", partition));
}

Example #20

Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0

5 votes

private long getOffset(KafkaPartition partition, Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo)
    throws KafkaOffsetRetrievalFailureException {
  SimpleConsumer consumer = this.getSimpleConsumer(partition.getLeader().getHostAndPort());
  for (int i = 0; i < this.fetchOffsetRetries; i++) {
    try {
      OffsetResponse offsetResponse =
          consumer.getOffsetsBefore(new OffsetRequest(offsetRequestInfo, kafka.api.OffsetRequest.CurrentVersion(),
              this.clientName));
      if (offsetResponse.hasError()) {
        throw new RuntimeException("offsetReponse has error: "
            + offsetResponse.errorCode(partition.getTopicName(), partition.getId()));
      }
      return offsetResponse.offsets(partition.getTopicName(), partition.getId())[0];
    } catch (Exception e) {
      log.warn(String.format("Fetching offset for partition %s has failed %d time(s). Reason: %s", partition, i + 1,
          e));
      if (i < this.fetchOffsetRetries - 1) {
        try {
          Thread.sleep((long) ((i + Math.random()) * 1000));
        } catch (InterruptedException e2) {
          log.error("Caught interrupted exception between retries of getting latest offsets. " + e2);
        }
      }
    }
  }
  throw new KafkaOffsetRetrievalFailureException(String.format("Fetching offset for partition %s has failed.",
      partition));
}

Example #21

Source File: KafkaSimpleConsumer.java From Pistachio with Apache License 2.0

5 votes

public long getLastOffset() throws InterruptedException {
    OffsetResponse response = null;
    Broker previousLeader = leaderBroker;
    while (true) {
        TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
        Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
        requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
        kafka.javaapi.OffsetRequest request = new kafka.javaapi.OffsetRequest(
                requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);

        ensureConsumer(previousLeader);
        try {
            response = consumer.getOffsetsBefore(request);
        } catch (Exception e) {
            // e could be an instance of ClosedByInterruptException as SimpleConsumer.fetch uses nio
            if (Thread.interrupted()) {
                logger.info("catch exception of {} with interrupted in getLastOffset for {} - {}",
                        e.getClass().getName(), topic, partitionId);

                throw new InterruptedException();
            }
            logger.warn("caughte exception in getLastOffset {} - {}", topic, partitionId, e);
            response = null;
        }
        if (response == null || response.hasError()) {
            short errorCode = response != null ? response.errorCode(topic, partitionId) : ErrorMapping.UnknownCode();

            logger.warn("Error fetching data Offset for {} - {}, the Broker. Reason: {}",
                    topic, partitionId, errorCode);

            stopConsumer();
            previousLeader = leaderBroker;
            leaderBroker = null;
            continue;
        }
        break;
    }
    long[] offsets = response.offsets(topic, partitionId);
    return offsets[offsets.length - 1];
}

Example #22

Source File: KafkaSimpleConsumer.java From Pistachio with Apache License 2.0

5 votes

private long getOffset(boolean earliest) throws InterruptedException {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partitionId);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(
        earliest ? kafka.api.OffsetRequest.EarliestTime() : kafka.api.OffsetRequest.LatestTime(), 1));
    OffsetRequest request = new OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientId);
    OffsetResponse response = null;
    try {
        response = consumer.getOffsetsBefore(request);
    } catch (Exception e) {
        // e could be an instance of ClosedByInterruptException as SimpleConsumer.getOffsetsBefore uses nio
        if (Thread.interrupted()) {
            logger.info("catch exception of {} with interrupted in getOffset({}) for {} - {}",
                    e.getClass().getName(), earliest, topic, partitionId);

            throw new InterruptedException();
        }

        logger.error("caught exception in getOffsetsBefore {} - {}", topic, partitionId, e);
        return -1;
    }
    if (response.hasError()) {
        logger.error("error fetching data Offset from the Broker {}. reason: {}", leaderBroker.host(), response.errorCode(topic, partitionId));
        return -1;
    }
    long[] offsets = response.offsets(topic, partitionId);
    return earliest ? offsets[0] : offsets[offsets.length - 1];
}

Example #23

Source File: MetricSystemTest.java From eagle with Apache License 2.0

5 votes

private long getLastOffset(SimpleConsumer consumer, String topic, int partition, long whichTime, String clientName) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(whichTime, 1));
    OffsetRequest request = new OffsetRequest(requestInfo, kafka.api.OffsetRequest.CurrentVersion(), clientName);
    OffsetResponse response = consumer.getOffsetsBefore(request);
    if (response.hasError()) {
        System.out.println("error fetching data offset data the broker. reason: " + response.errorCode(topic, partition));
        return 0;
    }
    long[] offsets = response.offsets(topic, partition);
    return offsets[0];
}

Example #24

Source File: ScribeConsumer.java From Scribengin with GNU Affero General Public License v3.0

5 votes

private long getLatestOffsetFromKafka(String topic, int partition, long startTime) {
  TopicAndPartition tp = new TopicAndPartition(topic, partition);

  Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo =
      new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();

  requestInfo.put(tp, new PartitionOffsetRequestInfo(startTime, 1));

  OffsetRequest req = new OffsetRequest(
      requestInfo, kafka.api.OffsetRequest.CurrentVersion(), getClientName());

  OffsetResponse resp = consumer.getOffsetsBefore(req);

  if (resp.hasError()) {
    LOG.error("error when fetching offset: " + resp.errorCode(topic, partition)); //xxx
    // In case you wonder what the error code really means.
    // System.out.println("OffsetOutOfRangeCode()" + ErrorMapping.OffsetOutOfRangeCode());
    // System.out.println("BrokerNotAvailableCode()" + ErrorMapping.BrokerNotAvailableCode());
    // System.out.println("InvalidFetchSizeCode()" + ErrorMapping.InvalidFetchSizeCode());
    // System.out.println("InvalidMessageCode()" + ErrorMapping.InvalidMessageCode());
    // System.out.println("LeaderNotAvailableCode()" + ErrorMapping.LeaderNotAvailableCode());
    // System.out.println("MessageSizeTooLargeCode()" + ErrorMapping.MessageSizeTooLargeCode());
    // System.out.println("NotLeaderForPartitionCode()" + ErrorMapping.NotLeaderForPartitionCode());
    // System.out.println("OffsetMetadataTooLargeCode()" + ErrorMapping.OffsetMetadataTooLargeCode());
    // System.out.println("ReplicaNotAvailableCode()" + ErrorMapping.ReplicaNotAvailableCode());
    // System.out.println("RequestTimedOutCode()" + ErrorMapping.RequestTimedOutCode());
    // System.out.println("StaleControllerEpochCode()" + ErrorMapping.StaleControllerEpochCode());
    // System.out.println("UnknownCode()" + ErrorMapping.UnknownCode());
    // System.out.println("UnknownTopicOrPartitionCode()" + ErrorMapping.UnknownTopicOrPartitionCode());

    //LOG.error("error when fetching offset: " + resp.errorcode(topic, partition));
    return 0;
  }

  return resp.offsets(topic, partition)[0];
}

Example #25

Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0

5 votes

@Override
public long getEarliestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
      Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
          new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.EarliestTime(), 1));
  return getOffset(partition, offsetRequestInfo);
}

Example #26

Source File: Kafka08ConsumerClient.java From incubator-gobblin with Apache License 2.0

5 votes

@Override
public long getLatestOffset(KafkaPartition partition) throws KafkaOffsetRetrievalFailureException {
  Map<TopicAndPartition, PartitionOffsetRequestInfo> offsetRequestInfo =
      Collections.singletonMap(new TopicAndPartition(partition.getTopicName(), partition.getId()),
          new PartitionOffsetRequestInfo(kafka.api.OffsetRequest.LatestTime(), 1));
  return getOffset(partition, offsetRequestInfo);
}

Example #27

Source File: KafkaUtils.java From storm-kafka-0.8-plus with Apache License 2.0

5 votes

public static long getOffset(SimpleConsumer consumer, String topic, int partition, long startOffsetTime) {
    TopicAndPartition topicAndPartition = new TopicAndPartition(topic, partition);
    Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<TopicAndPartition, PartitionOffsetRequestInfo>();
    requestInfo.put(topicAndPartition, new PartitionOffsetRequestInfo(startOffsetTime, 1));
    OffsetRequest request = new OffsetRequest(
            requestInfo, kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

    long[] offsets = consumer.getOffsetsBefore(request).offsets(topic, partition);
    if (offsets.length > 0) {
        return offsets[0];
    } else {
        return NO_OFFSET;
    }
}

Example #28

Source File: SimpleKafkaConsumer.java From twill with Apache License 2.0

5 votes

/**
 * Retrieves the last offset before the given timestamp for a given topic partition.
 *
 * @return The last offset before the given timestamp or {@code 0} if failed to do so.
 */
private long getLastOffset(TopicPartition topicPart, long timestamp) {
  BrokerInfo brokerInfo = brokerService.getLeader(topicPart.getTopic(), topicPart.getPartition());
  SimpleConsumer consumer = brokerInfo == null ? null : consumers.getUnchecked(brokerInfo);

  // If no broker, treat it as failure attempt.
  if (consumer == null) {
    LOG.warn("Failed to talk to any broker. Default offset to 0 for {}", topicPart);
    return 0L;
  }

  // Fire offset request
  OffsetRequest request = new OffsetRequest(ImmutableMap.of(
    new TopicAndPartition(topicPart.getTopic(), topicPart.getPartition()),
    new PartitionOffsetRequestInfo(timestamp, 1)
  ), kafka.api.OffsetRequest.CurrentVersion(), consumer.clientId());

  OffsetResponse response = consumer.getOffsetsBefore(request);

  // Retrieve offsets from response
  long[] offsets = response.hasError() ? null : response.offsets(topicPart.getTopic(), topicPart.getPartition());
  if (offsets == null || offsets.length <= 0) {
    short errorCode = response.errorCode(topicPart.getTopic(), topicPart.getPartition());

    // If the topic partition doesn't exists, use offset 0 without logging error.
    if (errorCode != ErrorMapping.UnknownTopicOrPartitionCode()) {
      consumers.refresh(brokerInfo);
      LOG.warn("Failed to fetch offset for {} with timestamp {}. Error: {}. Default offset to 0.",
               topicPart, timestamp, errorCode);
    }
    return 0L;
  }

  LOG.debug("Offset {} fetched for {} with timestamp {}.", offsets[0], topicPart, timestamp);
  return offsets[0];
}

Example #29

Source File: SimpleConsumerThread.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Request offsets before a specific time for a set of partitions, via a Kafka consumer.
 *
 * @param consumer The consumer connected to lead broker
 * @param partitions The list of partitions we need offsets for
 * @param whichTime The type of time we are requesting. -1 and -2 are special constants (See OffsetRequest)
 */
private static void requestAndSetSpecificTimeOffsetsFromKafka(
		SimpleConsumer consumer,
		List<KafkaTopicPartitionState<TopicAndPartition>> partitions,
		long whichTime) throws IOException {
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
	for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) {
		requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(whichTime, 1));
	}

	requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo);
}

Example #30

Source File: SimpleConsumerThread.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * For a set of partitions, if a partition is set with the special offsets {@link OffsetRequest#EarliestTime()}
 * or {@link OffsetRequest#LatestTime()}, replace them with actual offsets requested via a Kafka consumer.
 *
 * @param consumer The consumer connected to lead broker
 * @param partitions The list of partitions we need offsets for
 */
private static void requestAndSetEarliestOrLatestOffsetsFromKafka(
		SimpleConsumer consumer,
		List<KafkaTopicPartitionState<TopicAndPartition>> partitions) throws Exception {
	Map<TopicAndPartition, PartitionOffsetRequestInfo> requestInfo = new HashMap<>();
	for (KafkaTopicPartitionState<TopicAndPartition> part : partitions) {
		if (part.getOffset() == OffsetRequest.EarliestTime() || part.getOffset() == OffsetRequest.LatestTime()) {
			requestInfo.put(part.getKafkaPartitionHandle(), new PartitionOffsetRequestInfo(part.getOffset(), 1));
		}
	}

	requestAndSetOffsetsFromKafka(consumer, partitions, requestInfo);
}