Java Code Examples for org.apache.kafka.streams.StreamsBuilder#globalTable()

The following examples show how to use org.apache.kafka.streams.StreamsBuilder#globalTable() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractKafkaRepository.java    From SkaETL with Apache License 2.0 8 votes vote down vote up
public AbstractKafkaRepository(String name, Serde<V> valueSerde, Function<V,String> keyFunction, KafkaAdminService kafkaAdminService, KafkaConfiguration kafkaConfiguration) {
    this.repositoryName = name + "-db";
    this.keyFunction = keyFunction;
    this.producer = KafkaUtils.kafkaProducer(kafkaConfiguration.getBootstrapServers(), StringSerializer.class, JsonNodeSerialializer.class);
    kafkaAdminService.createTopic(kafkaAdminService.buildTopicInfo(repositoryName,TopicConfig.CLEANUP_POLICY_COMPACT));

    Properties props = KafkaUtils.createKStreamProperties(repositoryName + "-stream"+ UUID.randomUUID().toString(), kafkaConfiguration.getBootstrapServers());
    StreamsBuilder builder = new StreamsBuilder();

    final GlobalKTable<String, V> globalKTable = builder.globalTable(repositoryName, materialize(valueSerde));

    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    producer.flush();
    keyValueStore = streams.store(getStoreName(), QueryableStoreTypes.keyValueStore());

    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

}
 
Example 2
Source File: EmailService.java    From qcon-microservices with Apache License 2.0 8 votes vote down vote up
private KafkaStreams processStreams(final String bootstrapServers, final String stateDir) throws IOException {

        final StreamsBuilder builder = new StreamsBuilder();

        //Create the streams/tables for the join
        final KStream<String, Order> orders = builder.stream(ORDERS.name(),
                Consumed.with(ORDERS.keySerde(), ORDERS.valueSerde()));
        final GlobalKTable<String, Customer> customers = builder.globalTable(CUSTOMERS.name(),
                Consumed.with(CUSTOMERS.keySerde(), CUSTOMERS.valueSerde()));
//
        // Join a stream and a table then send an email for each
        // GlobalKTable to stream join takes three arguments: Table, mapping of stream (key,value) to table key for join
        // And the join function - takes values from stream and table and returns result
        orders.join(customers,
                        (orderID, order) -> order.getCustomerId(),
                (order, customer) -> new EmailTuple(order,customer))
                //Now for each tuple send an email.
                .peek((key, emailTuple)
                        -> emailer.sendEmail(emailTuple)
                );

        return new KafkaStreams(builder.build(), configStreams(bootstrapServers, stateDir, SERVICE_APP_ID));
    }
 
Example 3
Source File: NameJoinGlobalKTable.java    From fluent-kafka-streams-tests with MIT License 7 votes vote down vote up
public Topology getTopologyWithIntermediateTopic() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, Long> inputStream =
            builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Long(), Serdes.Long()));

    builder.stream(NAME_INPUT, Consumed.with(Serdes.Long(), Serdes.String()))
            .mapValues(name -> name.toUpperCase())
            .to(INTERMEDIATE_TOPIC);

    final GlobalKTable<Long, String> joinTable = builder.globalTable(INTERMEDIATE_TOPIC);

    inputStream
            .join(joinTable,
                    (id, valueId) -> valueId,
                    (id, name) -> name)
            .to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.String()));

    return builder.build();
}
 
Example 4
Source File: NameJoinGlobalKTable.java    From fluent-kafka-streams-tests with MIT License 6 votes vote down vote up
public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Long, Long> inputStream =
            builder.stream(INPUT_TOPIC, Consumed.with(Serdes.Long(), Serdes.Long()));

    final GlobalKTable<Long, String> joinTable = builder.globalTable(NAME_INPUT);

    inputStream
            .join(joinTable,
                    (id, valueId) -> valueId,
                    (id, name) -> name)
            .to(OUTPUT_TOPIC, Produced.with(Serdes.Long(), Serdes.String()));

    return builder.build();
}
 
Example 5
Source File: TopologyProducer.java    From quarkus-quickstarts with Apache License 2.0 5 votes vote down vote up
@Produces
public Topology buildTopology() {
    StreamsBuilder builder = new StreamsBuilder();

    JsonbSerde<WeatherStation> weatherStationSerde = new JsonbSerde<>(WeatherStation.class);
    JsonbSerde<Aggregation> aggregationSerde = new JsonbSerde<>(Aggregation.class);

    KeyValueBytesStoreSupplier storeSupplier = Stores.persistentKeyValueStore(WEATHER_STATIONS_STORE);

    GlobalKTable<Integer, WeatherStation> stations = builder.globalTable(
            WEATHER_STATIONS_TOPIC,
            Consumed.with(Serdes.Integer(), weatherStationSerde));

    builder.stream(
            TEMPERATURE_VALUES_TOPIC,
            Consumed.with(Serdes.Integer(), Serdes.String()))
            .join(
                    stations,
                    (stationId, timestampAndValue) -> stationId,
                    (timestampAndValue, station) -> {
                        String[] parts = timestampAndValue.split(";");
                        return new TemperatureMeasurement(station.id, station.name, Instant.parse(parts[0]),
                                Double.valueOf(parts[1]));
                    })
            .groupByKey()
            .aggregate(
                    Aggregation::new,
                    (stationId, value, aggregation) -> aggregation.updateFrom(value),
                    Materialized.<Integer, Aggregation> as(storeSupplier)
                            .withKeySerde(Serdes.Integer())
                            .withValueSerde(aggregationSerde))
            .toStream()
            .to(
                    TEMPERATURES_AGGREGATED_TOPIC,
                    Produced.with(Serdes.Integer(), aggregationSerde));

    return builder.build();
}
 
Example 6
Source File: AbstractKafkaStreamsBinderProcessor.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private <K, V> GlobalKTable<K, V> materializedAsGlobalKTable(
		StreamsBuilder streamsBuilder, String destination, String storeName,
		Serde<K> k, Serde<V> v, Topology.AutoOffsetReset autoOffsetReset, KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties) {
	final Consumed<K, V> consumed = getConsumed(kafkaStreamsConsumerProperties, k, v, autoOffsetReset);
	return streamsBuilder.globalTable(
			this.bindingServiceProperties.getBindingDestination(destination),
			consumed,
			getMaterialized(storeName, k, v));
}
 
Example 7
Source File: AbstractKafkaStreamsBinderProcessor.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
private GlobalKTable<?, ?> getGlobalKTable(KafkaStreamsConsumerProperties kafkaStreamsConsumerProperties,
		StreamsBuilder streamsBuilder,
		Serde<?> keySerde, Serde<?> valueSerde, String materializedAs,
		String bindingDestination, Topology.AutoOffsetReset autoOffsetReset) {
	final Consumed<?, ?> consumed = getConsumed(kafkaStreamsConsumerProperties, keySerde, valueSerde, autoOffsetReset);
	return materializedAs != null
			? materializedAsGlobalKTable(streamsBuilder, bindingDestination,
			materializedAs, keySerde, valueSerde, autoOffsetReset, kafkaStreamsConsumerProperties)
			: streamsBuilder.globalTable(bindingDestination,
			consumed);
}
 
Example 8
Source File: CustomSinkApp.java    From Kafka-Streams-Real-time-Stream-Processing with The Unlicense 4 votes vote down vote up
public static void main(String[] args) {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
    props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreLocation);

    StreamsBuilder streamsBuilder = new StreamsBuilder();

    //Global table for mappings
    GlobalKTable<String, TableMap> topicTableMapGlobalKTable = streamsBuilder.globalTable(
        AppConfigs.topicTableMap,
        Consumed.with(AppSerdes.String(),
            AppSerdes.TableMap()));

    //Stream of Records
    KStream<String, GenericRecord> recordKStream = streamsBuilder.stream(
        AppConfigs.topicPattern,
        Consumed.with(AppSerdes.String(),
            AppSerdes.GenericRecord())
    ).transform(() -> new RecordTransformer());

    //Join to get Target Table Name
    KStream<String, GenericRecord> joinedKStream = recordKStream.join(
        topicTableMapGlobalKTable,
        (keyGenericRecord, valueGenericRecord) -> keyGenericRecord,
        (valueGenericRecord, valueTableMap) -> {
            valueGenericRecord.setAdditionalProperty(
                AppConfigs.targetTableField,
                valueTableMap.getTargetTable());
            return valueGenericRecord;
        }
    );

    //Change key to target table name and cleanup record
    KStream<String, GenericRecord> sinkRecord = joinedKStream.selectKey(
        (k, v) -> {
            String newKey = v.getAdditionalProperties()
                .get(AppConfigs.targetTableField);
            v.getAdditionalProperties().remove(AppConfigs.targetTableField);
            return newKey;
        }).peek((k, v) -> logger.info("Ready to Sink key= " + k + " value= " + v));

    //Sink to Target Database
    sinkRecord.process(() -> new SinkProcessor());

    //Start the stream and add a shutdown hook
    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), props);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 
Example 9
Source File: Top3NewsTypesDemo.java    From Kafka-Streams-Real-time-Stream-Processing with The Unlicense 4 votes vote down vote up
public static void main(String[] args) {
    Properties properties = new Properties();
    properties.put(StreamsConfig.APPLICATION_ID_CONFIG,
        AppConfigs.applicationID);
    properties.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
        AppConfigs.bootstrapServers);
    properties.put(StreamsConfig.STATE_DIR_CONFIG,
        AppConfigs.stateStoreName);

    StreamsBuilder streamsBuilder = new StreamsBuilder();

    //Global table for all inventories
    GlobalKTable<String, AdInventories> adInventoriesGlobalKTable =
        streamsBuilder.globalTable(
            AppConfigs.inventoryTopic,
            Consumed.with(AppSerdes.String(),
                AppSerdes.AdInventories())
        );

    //Stream of Clicks
    KStream<String, AdClick> clicksKStream =
        streamsBuilder.stream(AppConfigs.clicksTopic,
            Consumed.with(AppSerdes.String(),
                AppSerdes.AdClick())
        );

    //Stream of Clicked Inventories
    KStream<String, AdInventories> clickedInventoryKStream =
        clicksKStream.join(
            adInventoriesGlobalKTable,
            (clickKey, clickValue) -> clickKey,
            (adClick, adInventory) -> adInventory
        );

    //Group clicked inventories on NewsType and count
    //You will get clicks by news Types
    KTable<String, Long> clicksByNewsTypeKTable =
        clickedInventoryKStream.groupBy(
            (inventoryID, inventoryValue) -> inventoryValue.getNewsType(),
            Grouped.with(AppSerdes.String(), AppSerdes.AdInventories())
        ).count();

    //The clicksByNewsType is exhaustive and distributed
    //We need to compute the top 3 only
    //In order to do that, we must following
    //1. Bring all clicksByNewsType to a single partition
    //2. Sort them by clicks and take only top 3
    //There are two steps to achieve this.
    //1. Group them on a common key to bring it to a single partition
    //2. Simulate Sort()+Top(3) using a custom aggregation
    KTable<String, Top3NewsTypes> top3NewsTypesKTable =
        clicksByNewsTypeKTable.groupBy(
            (k_newsType, v_clickCount) -> {
                ClicksByNewsType value = new ClicksByNewsType();
                value.setNewsType(k_newsType);
                value.setClicks(v_clickCount);
                return KeyValue.pair(AppConfigs.top3AggregateKey, value);
            },
            Grouped.with(AppSerdes.String(), AppSerdes.ClicksByNewsType())
        ).aggregate(Top3NewsTypes::new,
            (k, newClickByNewsType, aggTop3NewType) -> {
                aggTop3NewType.add(newClickByNewsType);
                return aggTop3NewType;
            },
            (k, oldClickByNewsType, aggTop3NewType) -> {
                aggTop3NewType.remove(oldClickByNewsType);
                return aggTop3NewType;
            },
            Materialized.<String, Top3NewsTypes, KeyValueStore<Bytes, byte[]>>
                as("top3-clicks")
                .withKeySerde(AppSerdes.String())
                .withValueSerde(AppSerdes.Top3NewsTypes()));

    top3NewsTypesKTable.toStream().foreach((k, v) -> {
        try {
            logger.info("k=" + k + " v= " + v.getTop3Sorted());
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        }
    });

    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), properties);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}
 
Example 10
Source File: GlobalKTableExample.java    From kafka-streams-in-action with Apache License 2.0 2 votes vote down vote up
public static void main(String[] args) throws Exception {


        StreamsConfig streamsConfig = new StreamsConfig(getProperties());

        Serde<String> stringSerde = Serdes.String();
        Serde<StockTransaction> transactionSerde = StreamsSerdes.StockTransactionSerde();
        Serde<TransactionSummary> transactionSummarySerde = StreamsSerdes.TransactionSummarySerde();


        StreamsBuilder builder = new StreamsBuilder();
        long twentySeconds = 1000 * 20;

        KeyValueMapper<Windowed<TransactionSummary>, Long, KeyValue<String, TransactionSummary>> transactionMapper = (window, count) -> {
            TransactionSummary transactionSummary = window.key();
            String newKey = transactionSummary.getIndustry();
            transactionSummary.setSummaryCount(count);
            return KeyValue.pair(newKey, transactionSummary);
        };

        KStream<String, TransactionSummary> countStream =
                builder.stream( STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, transactionSerde).withOffsetResetPolicy(LATEST))
                        .groupBy((noKey, transaction) -> TransactionSummary.from(transaction), Serialized.with(transactionSummarySerde, transactionSerde))
                        .windowedBy(SessionWindows.with(twentySeconds)).count()
                        .toStream().map(transactionMapper);

        GlobalKTable<String, String> publicCompanies = builder.globalTable(COMPANIES.topicName());
        GlobalKTable<String, String> clients = builder.globalTable(CLIENTS.topicName());


        countStream.leftJoin(publicCompanies, (key, txn) -> txn.getStockTicker(),TransactionSummary::withCompanyName)
                .leftJoin(clients, (key, txn) -> txn.getCustomerId(), TransactionSummary::withCustomerName)
                .print(Printed.<String, TransactionSummary>toSysOut().withLabel("Resolved Transaction Summaries"));


        
        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        kafkaStreams.cleanUp();


        kafkaStreams.setUncaughtExceptionHandler((t, e) -> {
            LOG.error("had exception ", e);
        });

        CustomDateGenerator dateGenerator = CustomDateGenerator.withTimestampsIncreasingBy(Duration.ofMillis(750));

        DataGenerator.setTimestampGenerator(dateGenerator::get);

        MockDataProducer.produceStockTransactions(2, 5, 3, true);

        LOG.info("Starting GlobalKTable Example");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
        Thread.sleep(65000);
        LOG.info("Shutting down the GlobalKTable Example Application now");
        kafkaStreams.close();
        MockDataProducer.shutdown();
    }