• Search by Module
• Search by Words
• Search Projects

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• django-logpipe-master
  • src
    • logpipe
      • consumer.py
      • exceptions.py
      • config.py
      • backend
        • kinesis.py
        • kafka.py
        • __init__.py
      • models.py
      • locale
        • es
          • LC_MESSAGES
            • django.mo
            • django.po
      • constants.py
      • registry.py
      • __init__.py
      • migrations
        • 0001_initial.py
        • 0003_auto_20170427_1703.py
        • 0002_auto_20170427_1451.py
        • __init__.py
        • 0005_auto_20180917_1348.py
        • 0004_auto_20170502_1403.py
      • tests
        • integration
          • test_roundtrip.py
          • __init__.py
        • common.py
        • __init__.py
        • unit
          • kafka
            • test_producer.py
            • test_consumer.py
            • __init__.py
          • kinesis
            • test_producer.py
            • test_consumer.py
            • __init__.py
          • test_format.py
          • test_producer.py
          • test_settings.py
          • __init__.py
      • settings.py
      • admin.py
      • format.py
      • management
        • commands
          • __init__.py
          • run_kafka_consumer.py
        • __init__.py
      • producer.py
      • formats
        • pickle.py
        • __init__.py
        • msgpack.py
        • json.py
  • Makefile
  • LICENSE
  • README.rst
  • setup.py
  • setup.cfg
  • requirements.txt
  • sandbox
    • manage.py
    • sandbox
      • urls.py
      • __init__.py
      • settings.py
    • lptester
      • apps.py
      • serializers.py
      • models.py
      • constants.py
      • consumers.py
      • __init__.py
      • migrations
        • 0001_initial.py
        • __init__.py
        • 0002_person_uuid.py
      • admin.py
      • signals.py
      • producers.py
  • Dockerfile
  • .gitlab-ci.yml
  • .gitignore
  • MANIFEST.in
  • docker-compose.yml
  • tox.ini

from django.apps import apps
from ..exceptions import MissingTopicError
from .. import settings
from . import RecordMetadata, Record, get_offset_backend
import kafka
import logging


logger = logging.getLogger(__name__)



class ModelOffsetStore(object):
    def commit(self, consumer, message):
        KafkaOffset = apps.get_model(app_label='logpipe', model_name='KafkaOffset')
        logger.debug('Commit offset "%s" for topic "%s", partition "%s" to %s' % (
            message.offset, message.topic, message.partition, self.__class__.__name__))
        obj, created = KafkaOffset.objects.get_or_create(
            topic=message.topic,
            partition=message.partition)
        obj.offset = message.offset + 1
        obj.save()


    def seek(self, consumer, topic, partition):
        KafkaOffset = apps.get_model(app_label='logpipe', model_name='KafkaOffset')
        tp = kafka.TopicPartition(topic=topic, partition=partition)
        try:
            obj = KafkaOffset.objects.get(topic=topic, partition=partition)
            logger.debug('Seeking to offset "%s" on topic "%s", partition "%s"' % (obj.offset, topic, partition))
            consumer.client.seek(tp, obj.offset)
        except KafkaOffset.DoesNotExist:
            logger.debug('Seeking to beginning of topic "%s", partition "%s"' % (topic, partition))
            consumer.client.seek_to_beginning(tp)



class KafkaOffsetStore(object):
    def commit(self, consumer, message):
        logger.debug('Commit offset "%s" for topic "%s", partition "%s" to %s' % (
            message.offset, message.topic, message.partition, self.__class__.__name__))
        consumer.client.commit()

    def seek(self, consumer, topic, partition):
        pass



class Consumer(object):
    _client = None

    def __init__(self, topic_name, **kwargs):
        self.topic_name = topic_name
        self.client_kwargs = kwargs


    @property
    def client(self):
        if not self._client:
            kwargs = self._get_client_config()
            self._client = kafka.KafkaConsumer(**kwargs)
            tps = self._get_topic_partitions()
            self._client.assign(tps)
            backend = get_offset_backend()
            for tp in tps:
                backend.seek(self, tp.topic, tp.partition)
                self._client.committed(tp)
        return self._client


    def __iter__(self):
        return self


    def __next__(self):
        r = next(self.client)
        record = Record(
            topic=r.topic,
            partition=r.partition,
            offset=r.offset,
            timestamp=r.timestamp,
            key=r.key,
            value=r.value)
        return record


    def _get_topic_partitions(self):
        p = []
        partitions = self.client.partitions_for_topic(self.topic_name)
        if not partitions:
            raise MissingTopicError('Could not find topic %s. Does it exist?' % self.topic_name)
        for partition in partitions:
            tp = kafka.TopicPartition(self.topic_name, partition=partition)
            p.append(tp)
        return p


    def _get_client_config(self):
        kwargs = {
            'auto_offset_reset': 'earliest',
            'enable_auto_commit': False,
            'consumer_timeout_ms': 1000,
        }
        kwargs.update(settings.get('KAFKA_CONSUMER_KWARGS', {}))
        kwargs.update(self.client_kwargs)
        kwargs.update({
            'bootstrap_servers': settings.get('KAFKA_BOOTSTRAP_SERVERS'),
        })
        return kwargs



class Producer(object):
    _client = None


    @property
    def client(self):
        if not self._client:
            kwargs = self._get_client_config()
            self._client = kafka.KafkaProducer(**kwargs)
        return self._client


    def send(self, topic_name, key, value):
        key = key.encode()
        timeout = settings.get('KAFKA_SEND_TIMEOUT', 10)
        future = self.client.send(topic_name, key=key, value=value)
        metadata = future.get(timeout=timeout)
        return RecordMetadata(
            topic=topic_name,
            partition=metadata.partition,
            offset=metadata.offset)


    def _get_client_config(self):
        servers = settings.get('KAFKA_BOOTSTRAP_SERVERS')
        retries = settings.get('KAFKA_MAX_SEND_RETRIES', 0)
        return {
            'bootstrap_servers': servers,
            'retries': retries,
        }