org.apache.kafka.streams.state.WindowStore Java Examples

@Test
public void shouldCreateTumblingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 

@Test
public void shouldCreateHoppingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 

@StreamListener("input")
@KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000)
@SuppressWarnings({ "deprecation", "unchecked" })
public void process(KStream<Object, Product> input) {

	input.process(() -> new Processor<Object, Product>() {

		@Override
		public void init(ProcessorContext processorContext) {
			state = (WindowStore) processorContext.getStateStore("mystate");
		}

		@Override
		public void process(Object s, Product product) {
			processed = true;
		}

		@Override
		public void close() {
			if (state != null) {
				state.close();
			}
		}
	}, "mystate");
}
 

@StreamListener("input3")
@SuppressWarnings({"unchecked" })
public void process(KStream<Object, Product> input) {

	input.process(() -> new Processor<Object, Product>() {

		@Override
		public void init(ProcessorContext processorContext) {
			state = (WindowStore) processorContext.getStateStore("mystate");
		}

		@Override
		public void process(Object s, Product product) {
			processed = true;
		}

		@Override
		public void close() {
			if (state != null) {
				state.close();
			}
		}
	}, "mystate");
}
 

@Bean
public java.util.function.BiConsumer<KStream<Object, String>, KStream<Object, String>> process() {
	return (input0, input1) ->
			input0.process((ProcessorSupplier<Object, String>) () -> new Processor<Object, String>() {
				@Override
				@SuppressWarnings("unchecked")
				public void init(ProcessorContext context) {
					state1 = (KeyValueStore<Long, Long>) context.getStateStore("my-store");
					state2 = (WindowStore<Long, Long>) context.getStateStore("other-store");
				}

				@Override
				public void process(Object key, String value) {
					processed1 = true;
				}

				@Override
				public void close() {

				}
			}, "my-store", "other-store");
}
 

@Bean
public java.util.function.Consumer<KTable<Object, String>> hello() {
	return input -> {
		input.toStream().process(() -> new Processor<Object, String>() {
			@Override
			@SuppressWarnings("unchecked")
			public void init(ProcessorContext context) {
				state3 = (KeyValueStore<Long, Long>) context.getStateStore("my-store");
				state4 = (WindowStore<Long, Long>) context.getStateStore("other-store");
			}

			@Override
			public void process(Object key, String value) {
				processed2 = true;
			}

			@Override
			public void close() {

			}
		}, "my-store", "other-store");
	};
}
 

public Topology buildTopology(Properties envProps,
                              final SpecificAvroSerde<Click> clicksSerde) {
    final StreamsBuilder builder = new StreamsBuilder();

    final String inputTopic = envProps.getProperty("input.topic.name");
    final String outputTopic = envProps.getProperty("output.topic.name");

    // How long we "remember" an event.  During this time, any incoming duplicates of the event
    // will be, well, dropped, thereby de-duplicating the input data.
    //
    // The actual value depends on your use case.  To reduce memory and disk usage, you could
    // decrease the size to purge old windows more frequently at the cost of potentially missing out
    // on de-duplicating late-arriving records.
    final Duration windowSize = Duration.ofMinutes(2);

    // retention period must be at least window size -- for this use case, we don't need a longer retention period
    // and thus just use the window size as retention time
    final Duration retentionPeriod = windowSize;

    final StoreBuilder<WindowStore<String, Long>> dedupStoreBuilder = Stores.windowStoreBuilder(
            Stores.persistentWindowStore(storeName,
                    retentionPeriod,
                    windowSize,
                    false
            ),
            Serdes.String(),
            Serdes.Long());

    builder.addStateStore(dedupStoreBuilder);

    builder
            .stream(inputTopic, Consumed.with(Serdes.String(), clicksSerde))
            .transformValues(() -> new DeduplicationTransformer<>(windowSize.toMillis(), (key, value) -> value.getIp()), storeName)
            .filter((k, v) -> v != null)
            .to(outputTopic, Produced.with(Serdes.String(), clicksSerde));

    return builder.build();
}
 

@SuppressWarnings("unchecked")
public SchemaKTable aggregate(
    final Initializer initializer,
    final UdafAggregator aggregator,
    final WindowExpression windowExpression,
    final Serde<GenericRow> topicValueSerDe) {
  final KTable aggKtable;
  if (windowExpression != null) {
    final Materialized<String, GenericRow, ?> materialized
        = Materialized.<String, GenericRow, WindowStore<Bytes, byte[]>>with(
            Serdes.String(), topicValueSerDe);

    final KsqlWindowExpression ksqlWindowExpression = windowExpression.getKsqlWindowExpression();
    aggKtable = ksqlWindowExpression.applyAggregate(
        kgroupedStream,
        initializer,
        aggregator,
        materialized
    );
  } else {
    aggKtable = kgroupedStream.aggregate(
        initializer,
        aggregator,
        Materialized.with(Serdes.String(), topicValueSerDe)
    );
  }
  return new SchemaKTable(
      schema,
      aggKtable,
      keyField,
      sourceSchemaKStreams,
      windowExpression != null,
      SchemaKStream.Type.AGGREGATE,
      functionRegistry,
      schemaRegistryClient
  );

}
 

@StreamListener
@KafkaStreamsStateStore(name = "mystate", type = KafkaStreamsStateStoreProperties.StoreType.WINDOW, lengthMs = 300000, retentionMs = 300000)
@SuppressWarnings({ "deprecation", "unchecked" })
public void process(@Input("input1")KStream<Object, Product> input, @Input("input2")KStream<Object, Product> input2) {

	input.process(() -> new Processor<Object, Product>() {

		@Override
		public void init(ProcessorContext processorContext) {
			state = (WindowStore) processorContext.getStateStore("mystate");
		}

		@Override
		public void process(Object s, Product product) {
			processed = true;
		}

		@Override
		public void close() {
			if (state != null) {
				state.close();
			}
		}
	}, "mystate");

	//simple use of input2, we are not using input2 for anything other than triggering some test behavior.
	input2.foreach((key, value) -> { });
}
 

private void receiveAndValidate(ConfigurableApplicationContext context) throws Exception {
	Map<String, Object> senderProps = KafkaTestUtils.producerProps(embeddedKafka);
	DefaultKafkaProducerFactory<Integer, String> pf = new DefaultKafkaProducerFactory<>(senderProps);
	try {
		KafkaTemplate<Integer, String> template = new KafkaTemplate<>(pf, true);
		template.setDefaultTopic("words");
		template.sendDefault(1, "foobar");
		Thread.sleep(2000L);
		StateStoreTestApplication processorApplication = context
				.getBean(StateStoreTestApplication.class);

		KeyValueStore<Long, Long> state1 = processorApplication.state1;
		assertThat(processorApplication.processed1).isTrue();
		assertThat(state1 != null).isTrue();
		assertThat(state1.name()).isEqualTo("my-store");
		WindowStore<Long, Long> state2 = processorApplication.state2;
		assertThat(state2 != null).isTrue();
		assertThat(state2.name()).isEqualTo("other-store");
		assertThat(state2.persistent()).isTrue();

		KeyValueStore<Long, Long> state3 = processorApplication.state1;
		assertThat(processorApplication.processed2).isTrue();
		assertThat(state3 != null).isTrue();
		assertThat(state3.name()).isEqualTo("my-store");
		WindowStore<Long, Long> state4 = processorApplication.state2;
		assertThat(state4 != null).isTrue();
		assertThat(state4.name()).isEqualTo("other-store");
		assertThat(state4.persistent()).isTrue();
	}
	finally {
		pf.destroy();
	}
}
 

@Test void should_storeDependencies() {
  // Given: configs
  Duration dependenciesRetentionPeriod = Duration.ofMinutes(1);
  Duration dependenciesWindowSize = Duration.ofMillis(100);
  // When: topology created
  Topology topology = new DependencyStorageTopology(
      dependencyTopic,
      dependenciesRetentionPeriod,
      dependenciesWindowSize,
      true).get();
  TopologyDescription description = topology.describe();
  // Then: topology with 1 thread
  assertThat(description.subtopologies()).hasSize(1);
  // Given: streams configuration
  TopologyTestDriver testDriver = new TopologyTestDriver(topology, props);
  // When: a trace is passed
  ConsumerRecordFactory<String, DependencyLink> factory =
      new ConsumerRecordFactory<>(dependencyTopic, new StringSerializer(),
          dependencyLinkSerde.serializer());
  DependencyLink dependencyLink = DependencyLink.newBuilder()
      .parent("svc_a").child("svc_b").callCount(1).errorCount(0)
      .build();
  String dependencyLinkId = "svc_a:svc_b";
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink, 10L));
  WindowStore<String, DependencyLink> links = testDriver.getWindowStore(DEPENDENCIES_STORE_NAME);
  // Then: dependency link created
  WindowStoreIterator<DependencyLink> firstLink = links.fetch(dependencyLinkId, 0L, 100L);
  assertThat(firstLink).hasNext();
  assertThat(firstLink.next().value).isEqualTo(dependencyLink);
  // When: new links appear
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink, 90L));
  // Then: dependency link increases
  WindowStoreIterator<DependencyLink> secondLink = links.fetch(dependencyLinkId, 0L, 100L);
  assertThat(secondLink).hasNext();
  assertThat(secondLink.next().value.callCount()).isEqualTo(2);
  // When: time moves forward
  testDriver.advanceWallClockTime(dependenciesRetentionPeriod.toMillis() + 91L);
  testDriver.pipeInput(factory.create(dependencyTopic, dependencyLinkId, dependencyLink));
  // Then: dependency link is removed and restarted
  KeyValueIterator<Windowed<String>, DependencyLink> thirdLink = links.all();
  assertThat(thirdLink).hasNext();
  assertThat(thirdLink.next().value.callCount()).isEqualTo(1);
  // Close resources
  testDriver.close();
  dependencyLinkSerde.close();
}
 

@Override
@SuppressWarnings("unchecked")
public void init(final ProcessorContext context) {
    this.context = context;
    eventIdStore = (WindowStore<E, Long>) context.getStateStore(storeName);
}
 

private Materialized<Keys, AggregateFunction, WindowStore<Bytes, byte[]>> materializedMathOperationTimeWindow() {
    return Materialized.<Keys, AggregateFunction, WindowStore<Bytes, byte[]>>as("aggregated-stream-store")
            .withKeySerde(MetricsSerdes.keysSerde())
            .withValueSerde(MetricsSerdes.aggFunctionSerdes());
}
 

public static void main(String[] args) {

        if(args.length < 2){
            LOG.error("Need to specify host, port");
            System.exit(1);
        }

        String host = args[0];
        int port = Integer.parseInt(args[1]);
        final HostInfo hostInfo = new HostInfo(host, port);

        Properties properties = getProperties();
        properties.put(StreamsConfig.APPLICATION_SERVER_CONFIG, host+":"+port);

        StreamsConfig streamsConfig = new StreamsConfig(properties);
        Serde<String> stringSerde = Serdes.String();
        Serde<Long> longSerde = Serdes.Long();
        Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde();
        WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerde.serializer());
        WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringSerde.deserializer());
        Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer, windowedDeserializer);
        Serde<CustomerTransactions> customerTransactionsSerde = StreamsSerdes.CustomerTransactionsSerde();

        Aggregator<String, StockTransaction, Integer> sharesAggregator = (k, v, i) -> v.getShares() + i;

        StreamsBuilder builder = new StreamsBuilder();

        // data is already coming in keyed
        KStream<String, StockTransaction> stockTransactionKStream = builder.stream(MockDataProducer.STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, stockTransactionSerde)
                .withOffsetResetPolicy(Topology.AutoOffsetReset.LATEST));


        stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getSector(), v))
                .groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .count(Materialized.as("TransactionsBySector"))
                .toStream()
                .peek((k,v) -> LOG.info("Transaction count for {} {}", k, v))
                .to("sector-transaction-counts", Produced.with(stringSerde, longSerde));
        
        stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getCustomerId(), v))
                .groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(60)).until(TimeUnit.MINUTES.toMillis(120)))
                .aggregate(CustomerTransactions::new,(k, v, ct) -> ct.update(v),
                        (k, ct, other)-> ct.merge(other),
                        Materialized.<String, CustomerTransactions, SessionStore<Bytes, byte[]>>as("CustomerPurchaseSessions")
                                .withKeySerde(stringSerde).withValueSerde(customerTransactionsSerde))
                .toStream()
                .peek((k,v) -> LOG.info("Session info for {} {}", k, v))
                .to("session-transactions", Produced.with(windowedSerde, customerTransactionsSerde));


        stockTransactionKStream.groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .windowedBy(TimeWindows.of(10000))
                .aggregate(() -> 0, sharesAggregator,
                        Materialized.<String, Integer, WindowStore<Bytes, byte[]>>as("NumberSharesPerPeriod")
                                .withKeySerde(stringSerde)
                                .withValueSerde(Serdes.Integer()))
                .toStream().peek((k,v)->LOG.info("key is {} value is {}", k, v))
                .to("transaction-count", Produced.with(windowedSerde,Serdes.Integer()));


        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        InteractiveQueryServer queryServer = new InteractiveQueryServer(kafkaStreams, hostInfo);
        StateRestoreHttpReporter restoreReporter = new StateRestoreHttpReporter(queryServer);

        queryServer.init();

        kafkaStreams.setGlobalStateRestoreListener(restoreReporter);

        kafkaStreams.setStateListener(((newState, oldState) -> {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                LOG.info("Setting the query server to ready");
                queryServer.setReady(true);
            } else if (newState != KafkaStreams.State.RUNNING) {
                LOG.info("State not RUNNING, disabling the query server");
                queryServer.setReady(false);
            }
        }));

        kafkaStreams.setUncaughtExceptionHandler((t, e) -> {
            LOG.error("Thread {} had a fatal error {}", t, e, e);
            shutdown(kafkaStreams, queryServer);
        });


        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            shutdown(kafkaStreams, queryServer);
        }));

        LOG.info("Stock Analysis KStream Interactive Query App Started");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
    }
 

public static void main(String[] args) {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, AppConfigs.applicationID);
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, AppConfigs.bootstrapServers);
    props.put(StreamsConfig.STATE_DIR_CONFIG, AppConfigs.stateStoreName);

    StreamsBuilder streamsBuilder = new StreamsBuilder();
    KStream<String, SimpleInvoice> KS0 = streamsBuilder.stream(AppConfigs.posTopicName,
        Consumed.with(AppSerdes.String(), AppSerdes.SimpleInvoice())
            .withTimestampExtractor(new InvoiceTimeExtractor())
    );

    KGroupedStream<String, SimpleInvoice> KS1 = KS0.groupByKey(
        Grouped.with(AppSerdes.String(),
            AppSerdes.SimpleInvoice()));

    TimeWindowedKStream<String, SimpleInvoice> KS2 = KS1.windowedBy(
        TimeWindows.of(Duration.ofMinutes(5))
        //.grace(Duration.ofMillis(100))
    );

    KTable<Windowed<String>, Long> KT3 = KS2.count(
        //Materialized is not needed if you don't want to override defaults
        Materialized.<String, Long, WindowStore<Bytes, byte[]>>as("invoice-count")
        //.withRetention(Duration.ofHours(6))
    );

    //Suppress is only available in 2.1, Checkout 2.1 branch
    //.suppress(untilWindowCloses(unbounded()));


    KT3.toStream().foreach(
        (kWindowed, v) -> logger.info(
            "StoreID: " + kWindowed.key() +
                " Window start: " +
                Instant.ofEpochMilli(kWindowed.window().start())
                    .atOffset(ZoneOffset.UTC) +
                " Window end: " +
                Instant.ofEpochMilli(kWindowed.window().end())
                    .atOffset(ZoneOffset.UTC) +
                " Count: " + v +
                " Window#: " + kWindowed.window().hashCode()

        ));

    KafkaStreams streams = new KafkaStreams(streamsBuilder.build(), props);
    streams.start();
    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
}