org.apache.kafka.streams.kstream.TimeWindows Java Examples

The following examples show how to use org.apache.kafka.streams.kstream.TimeWindows. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: UserClicksPerMinute.java    From fluent-kafka-streams-tests with MIT License 8 votes vote down vote up
public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();
    final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.inputTopic);

    final KTable<Windowed<Integer>, Long> counts = clickEvents
            .groupByKey()
            .windowedBy(TimeWindows.of(Duration.ofMinutes(1)))
            .count();

    counts.toStream()
            .map((key, value) -> KeyValue.pair(
                    key.key(),
                    new ClickOutput(key.key(), value, key.window().start())))
            .to(this.outputTopic, Produced.with(Serdes.Integer(), new JsonSerde<>(ClickOutput.class)));

    return builder.build();
}
 
Example #2
Source File: KafkaStreamsBinderWordCountIntegrationTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@StreamListener
@SendTo("output")
public KStream<?, WordCount> process(
		@Input("input") KStream<Object, String> input) {

	return input
			.flatMapValues(
					value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null,
					new WordCount(key.key(), value,
							new Date(key.window().start()),
							new Date(key.window().end()))));
}
 
Example #3
Source File: WordCountMultipleBranchesIntegrationTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@StreamListener("input")
@SendTo({ "output1", "output2", "output3" })
@SuppressWarnings("unchecked")
public KStream<?, WordCount>[] process(KStream<Object, String> input) {

	Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english");
	Predicate<Object, WordCount> isFrench = (k, v) -> v.word.equals("french");
	Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish");

	return input
			.flatMapValues(
					value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.groupBy((key, value) -> value).windowedBy(TimeWindows.of(Duration.ofSeconds(5)))
			.count(Materialized.as("WordCounts-multi")).toStream()
			.map((key, value) -> new KeyValue<>(null,
					new WordCount(key.key(), value,
							new Date(key.window().start()),
							new Date(key.window().end()))))
			.branch(isEnglish, isFrench, isSpanish);
}
 
Example #4
Source File: KafkaStreamsBinderWordCountBranchesFunctionTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 6 votes vote down vote up
@Bean
@SuppressWarnings("unchecked")
public Function<KStream<Object, String>, KStream<?, WordCount>[]> process() {

	Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english");
	Predicate<Object, WordCount> isFrench = (k, v) -> v.word.equals("french");
	Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish");

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.groupBy((key, value) -> value)
			.windowedBy(TimeWindows.of(5000))
			.count(Materialized.as("WordCounts-branch"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value,
					new Date(key.window().start()), new Date(key.window().end()))))
			.branch(isEnglish, isFrench, isSpanish);
}
 
Example #5
Source File: MainVerticle.java    From kiqr with Apache License 2.0 6 votes vote down vote up
@Override
public void start(Future<Void> startFuture) throws Exception {
    Properties props = new Properties();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, "kiqr");
    props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class);
    props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.LongSerde.class);
    props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, "0");

    KStreamBuilder builder = new KStreamBuilder();
    KTable<String, Long> table = builder.table(Serdes.String(), Serdes.Long(), "visits", "visitStore");
    KTable<Windowed<String>, Long> windowedCount = table.toStream().groupByKey().count(TimeWindows.of(60), "visitCount");





    vertx.deployVerticle(RestKiqrServerVerticle.Builder.serverBuilder(builder, props).withPort(2901).build(), res -> {
        if (res.succeeded()) {
            startFuture.complete();
        } else {
            startFuture.fail(res.cause());
        }
    });
}
 
Example #6
Source File: KafkaStreamsBranchingSample.java    From spring-cloud-stream-samples with Apache License 2.0 6 votes vote down vote up
@Bean
@SuppressWarnings("unchecked")
public Function<KStream<Object, String>, KStream<?, WordCount>[]> process() {

	Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english");
	Predicate<Object, WordCount> isFrench =  (k, v) -> v.word.equals("french");
	Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish");

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.groupBy((key, value) -> value)
			.windowedBy(TimeWindows.of(Duration.ofSeconds(6)))
			.count(Materialized.as("WordCounts-1"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null,
					new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))))
			.branch(isEnglish, isFrench, isSpanish);
}
 
Example #7
Source File: HoppingWindowExpressionTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldCreateHoppingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 
Example #8
Source File: TumblingWindowExpressionTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
@Test
public void shouldCreateTumblingWindowAggregate() {
  final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class);
  final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class);
  final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class);
  final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS);
  final Initializer initializer = () -> 0;
  final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store");

  EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream);
  EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null);
  EasyMock.replay(stream, windowedKStream);

  windowExpression.applyAggregate(stream, initializer, aggregator, store);
  EasyMock.verify(stream, windowedKStream);
}
 
Example #9
Source File: KafkaStreamsProductTrackerApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, Product>, KStream<Integer, ProductStatus>> process() {
	return input -> input
			.filter((key, product) -> productIds().contains(product.getId()))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class)))
			.windowedBy(TimeWindows.of(Duration.ofSeconds(60)))
			.count(Materialized.as("product-counts"))
			.toStream()
			.map((key, value) -> new KeyValue<>(key.key().id, new ProductStatus(key.key().id,
					value, Instant.ofEpochMilli(key.window().start()).atZone(ZoneId.systemDefault()).toLocalTime(),
					Instant.ofEpochMilli(key.window().end()).atZone(ZoneId.systemDefault()).toLocalTime())));
}
 
Example #10
Source File: KafkaStreamsDlqSample.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, String>,KStream<?, WordCount>> process() {

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(5000))
			.count(Materialized.as("WordCounts-1"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null,
					new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
}
 
Example #11
Source File: KafkaStreamsWordCountApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Bytes, String>, KStream<Bytes, WordCount>> process() {

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofMillis(WINDOW_SIZE_MS)))
			.count(Materialized.as("WordCounts-1"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
}
 
Example #12
Source File: KafkaStreamsWordCountApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, String>, KStream<?, WordCount>> process() {

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofSeconds(60)))
			.count(Materialized.as("WordCounts-1"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null,
					new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
}
 
Example #13
Source File: KafkaStreamsBinderWordCountFunctionTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, String>, KStream<String, WordCount>> process() {

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(5000))
			.count(Materialized.as("foo-WordCounts"))
			.toStream()
			.map((key, value) -> new KeyValue<>(key.key(), new WordCount(key.key(), value,
					new Date(key.window().start()), new Date(key.window().end()))));
}
 
Example #14
Source File: KafkaStreamsWordCountApplication.java    From spring-cloud-stream-samples with Apache License 2.0 5 votes vote down vote up
@Bean
public Function<KStream<Object, String>, KStream<?, WordCount>> process() {

	return input -> input
			.flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofSeconds(20)))
			.count(Materialized.as("WordCounts-1"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end()))));
}
 
Example #15
Source File: KafkaStreamsNativeEncodingDecodingTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@StreamListener("input")
@SendTo("output")
public KStream<?, String> process(KStream<Object, String> input) {

	return input
			.flatMapValues(
					value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Serialized.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts-x"))
			.toStream().map((key, value) -> new KeyValue<>(null,
					"Count for " + key.key() + " : " + value));
}
 
Example #16
Source File: StocksKafkaStreamsDriver.java    From kafka-streams with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {

        StreamsConfig streamingConfig = new StreamsConfig(getProperties());

        JsonSerializer<StockTransactionCollector> stockTransactionsSerializer = new JsonSerializer<>();
        JsonDeserializer<StockTransactionCollector> stockTransactionsDeserializer = new JsonDeserializer<>(StockTransactionCollector.class);
        JsonDeserializer<StockTransaction> stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class);
        JsonSerializer<StockTransaction> stockTxnJsonSerializer = new JsonSerializer<>();
        Serde<StockTransaction> transactionSerde = Serdes.serdeFrom(stockTxnJsonSerializer,stockTxnDeserializer);
        StringSerializer stringSerializer = new StringSerializer();
        StringDeserializer stringDeserializer = new StringDeserializer();
        Serde<String> stringSerde = Serdes.serdeFrom(stringSerializer,stringDeserializer);
        Serde<StockTransactionCollector> collectorSerde = Serdes.serdeFrom(stockTransactionsSerializer,stockTransactionsDeserializer);
        WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerializer);
        WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringDeserializer);
        Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer,windowedDeserializer);

        KStreamBuilder kStreamBuilder = new KStreamBuilder();


        KStream<String,StockTransaction> transactionKStream =  kStreamBuilder.stream(stringSerde,transactionSerde,"stocks");

        transactionKStream.map((k,v)-> new KeyValue<>(v.getSymbol(),v))
                          .through(stringSerde, transactionSerde,"stocks-out")
                          .groupBy((k,v) -> k, stringSerde, transactionSerde)
                          .aggregate(StockTransactionCollector::new,
                               (k, v, stockTransactionCollector) -> stockTransactionCollector.add(v),
                               TimeWindows.of(10000),
                               collectorSerde, "stock-summaries")
                .to(windowedSerde,collectorSerde,"transaction-summary");


        System.out.println("Starting StockStreams Example");
        KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder,streamingConfig);
        kafkaStreams.start();
        System.out.println("Now started StockStreams Example");

    }
 
Example #17
Source File: OutboundValueNullSkippedConversionTest.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@StreamListener
@SendTo("output")
public KStream<?, KafkaStreamsBinderWordCountIntegrationTests.WordCount> process(
		@Input("input") KStream<Object, String> input) {

	return input
			.flatMapValues(
					value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Serialized.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts"))
			.toStream()
			.map((key, value) -> new KeyValue<>(null, null));
}
 
Example #18
Source File: DeserializationErrorHandlerByKafkaTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@StreamListener("input")
@SendTo("output")
public KStream<?, String> process(KStream<Object, String> input) {

	return input
			.flatMapValues(
					value -> Arrays.asList(value.toLowerCase().split("\\W+")))
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Serialized.with(Serdes.String(), Serdes.String()))
			.windowedBy(TimeWindows.of(5000)).count(Materialized.as("foo-WordCounts-x"))
			.toStream().map((key, value) -> new KeyValue<>(null,
					"Count for " + key.key() + " : " + value));
}
 
Example #19
Source File: ErrorEventsPerMinute.java    From fluent-kafka-streams-tests with MIT License 5 votes vote down vote up
public Topology getTopology() {
    final StreamsBuilder builder = new StreamsBuilder();

    // Click Events
    final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.clickInputTopic,
            Consumed.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class)));

    final KTable<Windowed<Integer>, Long> counts = clickEvents
            .selectKey(((key, value) -> value.getStatus()))
            .filter(((key, value) -> key >= 400))
            .groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class)))
            .windowedBy(TimeWindows.of(Duration.ofMinutes(1)))  // 1 Minute in ms
            .count();

    // Status codes
    final KTable<Integer, StatusCode> statusCodes = builder.table(this.statusInputTopic,
            Consumed.with(Serdes.Integer(), new JsonSerde<>(StatusCode.class)));

    // Join
    final KStream<Integer, ErrorOutput> errors = counts.toStream()
            .map((key, value) -> KeyValue.pair(
                    key.key(),
                    new ErrorOutput(key.key(), value, key.window().start(), null /*empty definition*/)))
            .join(statusCodes,
                    (countRecord, code) -> new ErrorOutput(
                            countRecord.getStatusCode(), countRecord.getCount(), countRecord.getTime(), code.getDefinition()),
                    Joined.valueSerde(new JsonSerde<>(ErrorOutput.class)));
    errors.to(this.errorOutputTopic);

    // Send alert if more than 5x a certain error code per minute
    errors.filter((key, errorOutput) -> errorOutput.getCount() > 5L).to(this.alertTopic);

    return builder.build();
}
 
Example #20
Source File: DeserializtionErrorHandlerByBinderTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@StreamListener("input")
@SendTo("output")
public KStream<Integer, Long> process(KStream<Object, Product> input) {
	return input.filter((key, product) -> product.getId() == 123)
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Grouped.with(new JsonSerde<>(Product.class),
					new JsonSerde<>(Product.class)))
			.windowedBy(TimeWindows.of(5000))
			.count(Materialized.as("id-count-store-x")).toStream()
			.map((key, value) -> new KeyValue<>(key.key().id, value));
}
 
Example #21
Source File: KafkastreamsBinderPojoInputStringOutputIntegrationTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@StreamListener("input")
@SendTo("output")
public KStream<Integer, String> process(KStream<Object, Product> input) {

	return input.filter((key, product) -> product.getId() == 123)
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Serialized.with(new JsonSerde<>(Product.class),
					new JsonSerde<>(Product.class)))
			.windowedBy(TimeWindows.of(5000))
			.count(Materialized.as("id-count-store")).toStream()
			.map((key, value) -> new KeyValue<>(key.key().id,
					"Count for product with ID 123: " + value));
}
 
Example #22
Source File: KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java    From spring-cloud-stream-binder-kafka with Apache License 2.0 5 votes vote down vote up
@StreamListener("input")
@SendTo("output")
public KStream<Integer, Long> process(KStream<Object, Product> input) {
	return input.filter((key, product) -> product.getId() == 123)
			.map((key, value) -> new KeyValue<>(value, value))
			.groupByKey(Serialized.with(new JsonSerde<>(Product.class),
					new JsonSerde<>(Product.class)))
			.windowedBy(TimeWindows.of(5000))
			.count(Materialized.as("id-count-store-x")).toStream()
			.map((key, value) -> {
				return new KeyValue<>(key.key().id, value);
			});
}
 
Example #23
Source File: KafkaStreamingMain.java    From kafka-streams-api-websockets with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streaming-example");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
        props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
        props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
        props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1500);

//        To get data produced before process started
//        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
//        props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);

        KStreamBuilder builder = new KStreamBuilder();

        KStream<String, String> source = builder.stream("data-in");

        KStream<String, String> stats = source.groupByKey()
                .aggregate(KafkaStreamingStatistics::new,
                    (k, v, clusterstats) -> clusterstats.add(v),
                    TimeWindows.of(60000).advanceBy(10000),
                    Serdes.serdeFrom(new MySerde(), new MySerde()),
                    "data-store")
                .toStream((key, value) -> key.key().toString() + " " + key.window().start())
                .mapValues((job) -> job.computeAvgTime().toString());

        stats.to(Serdes.String(), Serdes.String(),  "data-out");

        KafkaStreams streams = new KafkaStreams(builder, props);

        streams.cleanUp();
        streams.start();

        Runtime.getRuntime().addShutdownHook(new Thread(streams::close));
    }
 
Example #24
Source File: SummaryBulkAggregation.java    From kafka-graphs with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public KTable<Windowed<Short>, T> run(final KStream<Edge<K>, EV> edgeStream) {

    //For parallel window support we key the edge stream by partition and apply a parallel fold per partition.
    //Finally, we merge all locally combined results into our final graph aggregation property.
    KTable<Windowed<Short>, S> partialAgg = edgeStream
        .groupByKey(Grouped.with(new KryoSerde<>(), new KryoSerde<>()))
        .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis)))
        .aggregate(this::initialValue, new PartialAgg<>(updateFun()))
        .toStream()
        .groupBy((k, v) -> GLOBAL_KEY)
        .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis)))
        .reduce(combineFun())
        .mapValues(aggregator(edgeStream), Materialized.<Windowed<Short>, S, KeyValueStore<Bytes, byte[]>>
            as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>()));

    if (transform() != null) {
        return partialAgg.mapValues(
            transform(),
            Materialized.<Windowed<Short>, T, KeyValueStore<Bytes, byte[]>>
                as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>())
        );
    }

    return (KTable<Windowed<Short>, T>) partialAgg;
}
 
Example #25
Source File: HoppingWindowExpression.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public KTable applyAggregate(
    KGroupedStream groupedStream,
    Initializer initializer,
    UdafAggregator aggregator,
    Materialized<String, GenericRow, ?> materialized
) {
  return groupedStream.windowedBy(
      TimeWindows.of(sizeUnit.toMillis(size))
          .advanceBy(advanceByUnit.toMillis(advanceBy))
  ).aggregate(initializer, aggregator, materialized);
}
 
Example #26
Source File: TumblingWindowExpression.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public KTable applyAggregate(final KGroupedStream groupedStream,
                             final Initializer initializer,
                             final UdafAggregator aggregator,
                             final Materialized<String, GenericRow, ?> materialized) {
  return groupedStream.windowedBy(TimeWindows.of(sizeUnit.toMillis(size)))
      .aggregate(initializer, aggregator, materialized);

}
 
Example #27
Source File: SpecificClientIntegrationITCase.java    From kiqr with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void produceMessages() throws Exception{
    Properties producerProps = new Properties();
    producerProps.put("bootstrap.servers", KAFKA_HOST + ":" + KAFKA_PORT);
    producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
    producerProps.put("value.serializer", "org.apache.kafka.common.serialization.LongSerializer");
    producerProps.put("linger.ms", 0);


    try(KafkaProducer<String, Long> producer = new KafkaProducer<>(producerProps)){
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key1", 1L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100L, "key1", 2L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100000L, "key1", 3L));


        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key2", 4L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100000L, "key2", 5L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100001L, "key2", 6L));

        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key3", 7L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 50000L, "key3", 8L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100001L, "key3", 9L));


        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key4", 10L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 1L, "key4", 11L));
        producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 2L, "key4", 12L));

    }

    CountDownLatch cdl = new CountDownLatch(12);


    Properties consumerProps = new Properties();
    consumerProps.put("bootstrap.servers",  KAFKA_HOST + ":" + KAFKA_PORT);
    consumerProps.put("group.id", UUID.randomUUID().toString());
    consumerProps.put("enable.auto.commit", "true");
    consumerProps.put("auto.offset.reset", "earliest");
    consumerProps.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
    consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.LongDeserializer");


    Runnable consumerRunnable = () -> {
        KafkaConsumer<String, Long> consumer = new KafkaConsumer<>(consumerProps);

        consumer.subscribe(Collections.singleton(TOPIC));

        int tryCount = 10;
        while(true){
            ConsumerRecords<String, Long> records = consumer.poll(500);
            records.forEach(rec -> cdl.countDown());

            tryCount--;
            if(cdl.getCount() == 0){
                consumer.close();
                return;
            } else if(tryCount == 0){
                throw new RuntimeException("times up");
            }
        }
    };

    consumerRunnable.run();

    cdl.await(10000, TimeUnit.MILLISECONDS);


    KStreamBuilder builder = new KStreamBuilder();
    KTable<String, Long> kv = builder.table(Serdes.String(), Serdes.Long(), TOPIC, "kv");

    kv.toStream().groupByKey().count(TimeWindows.of(10000L), "window");

    Properties streamProps = new Properties();
    streamProps.put(StreamsConfig.APPLICATION_ID_CONFIG, UUID.randomUUID().toString());
    streamProps.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,  KAFKA_HOST + ":" + KAFKA_PORT);
    streamProps.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0);
    streamProps.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName());
    streamProps.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName());

    CountDownLatch streamCdl = new CountDownLatch(2);


    RestKiqrServerVerticle.Builder verticleBuilder = RestKiqrServerVerticle.Builder.serverBuilder(builder, streamProps);
    RuntimeVerticle.Builder builder1 = verticleBuilder.withPort(44321).withStateListener((newState, oldState) -> {
        if (newState == KafkaStreams.State.RUNNING) streamCdl.countDown();
        System.out.println(oldState + " - " + newState);
    });

    AbstractVerticle verticle = verticleBuilder.build();

    CountDownLatch verticleCdl = new CountDownLatch(1);
    VERTX.deployVerticle(verticle, handler -> {
        verticleCdl.countDown();
    });

    streamCdl.await(100000, TimeUnit.MILLISECONDS);
    verticleCdl.await(100000, TimeUnit.MILLISECONDS);

}
 
Example #28
Source File: StatKStreamBuilderSupplier.java    From DBus with Apache License 2.0 4 votes vote down vote up
@Override
public KStreamBuilder get() {
    KStreamBuilder builder = new KStreamBuilder();
    KStream<String, String> stream = builder.stream((String[]) sources.toArray());

    KStream<HBKeySupplier.HBKey, String>[] streams =
            stream.filter((k, v) -> StringUtils.startsWith(v, "data_increment_heartbeat"))
                    .selectKey((k, v) -> new HBKeySupplier(k).get())
                    .filter((k, v) -> k.isNormalFormat)
                    .flatMapValues(v -> Arrays.asList("stat", "monitor"))
                    .branch((k, v) -> StringUtils.equalsIgnoreCase("stat", v),
                            (k, v) -> StringUtils.equalsIgnoreCase("monitor", v));

    streams[0].transform(StatTransformer::new).to(sink);
    KStream<String, PacketVo> monitor =
            streams[1].filterNot((k, v) -> !StringUtils.equalsIgnoreCase("abort", k.getStatus()))
                    .map((k, v) -> {
                        StringJoiner joiner = new StringJoiner("/");
                        joiner.add(HeartBeatConfigContainer.getInstance().getHbConf().getMonitorPath())
                                .add(k.getDs())
                                .add(StringUtils.equalsIgnoreCase(DbusDatasourceType.DB2.name(), k.getDbType()) ? StringUtils.upperCase(k.getSchema()) : k.getSchema())
                                .add(k.getTable()).add(String.valueOf(k.getPartition()));

                        String node = joiner.toString();
                        PacketVo packet = new PacketVo();
                        packet.setNode(node);
                        packet.setType("checkpoint");
                        packet.setTime(k.getCheckpointMs());
                        packet.setTxTime(k.getTxTimeMs());
                        return new KeyValue(node, packet);
                    });

    // 需要先进行shuff把key相同的分配到partition号
    monitor.through("monitor-repartition")
            .reduceByKey((agg, v) -> v.getTime() > agg.getTime() ? v : agg, TimeWindows.of("monitor", 2 * 60 * 1000))
            .toStream()
            .map((k, v) -> new KeyValue<>(k.key(), v))
            .process(new MonitorProcessorSupplier(), "zkInfo");

    return builder;
}
 
Example #29
Source File: StockPerformanceInteractiveQueryApplication.java    From kafka-streams-in-action with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) {

        if(args.length < 2){
            LOG.error("Need to specify host, port");
            System.exit(1);
        }

        String host = args[0];
        int port = Integer.parseInt(args[1]);
        final HostInfo hostInfo = new HostInfo(host, port);

        Properties properties = getProperties();
        properties.put(StreamsConfig.APPLICATION_SERVER_CONFIG, host+":"+port);

        StreamsConfig streamsConfig = new StreamsConfig(properties);
        Serde<String> stringSerde = Serdes.String();
        Serde<Long> longSerde = Serdes.Long();
        Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde();
        WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerde.serializer());
        WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringSerde.deserializer());
        Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer, windowedDeserializer);
        Serde<CustomerTransactions> customerTransactionsSerde = StreamsSerdes.CustomerTransactionsSerde();

        Aggregator<String, StockTransaction, Integer> sharesAggregator = (k, v, i) -> v.getShares() + i;

        StreamsBuilder builder = new StreamsBuilder();

        // data is already coming in keyed
        KStream<String, StockTransaction> stockTransactionKStream = builder.stream(MockDataProducer.STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, stockTransactionSerde)
                .withOffsetResetPolicy(Topology.AutoOffsetReset.LATEST));


        stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getSector(), v))
                .groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .count(Materialized.as("TransactionsBySector"))
                .toStream()
                .peek((k,v) -> LOG.info("Transaction count for {} {}", k, v))
                .to("sector-transaction-counts", Produced.with(stringSerde, longSerde));
        
        stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getCustomerId(), v))
                .groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(60)).until(TimeUnit.MINUTES.toMillis(120)))
                .aggregate(CustomerTransactions::new,(k, v, ct) -> ct.update(v),
                        (k, ct, other)-> ct.merge(other),
                        Materialized.<String, CustomerTransactions, SessionStore<Bytes, byte[]>>as("CustomerPurchaseSessions")
                                .withKeySerde(stringSerde).withValueSerde(customerTransactionsSerde))
                .toStream()
                .peek((k,v) -> LOG.info("Session info for {} {}", k, v))
                .to("session-transactions", Produced.with(windowedSerde, customerTransactionsSerde));


        stockTransactionKStream.groupByKey(Serialized.with(stringSerde, stockTransactionSerde))
                .windowedBy(TimeWindows.of(10000))
                .aggregate(() -> 0, sharesAggregator,
                        Materialized.<String, Integer, WindowStore<Bytes, byte[]>>as("NumberSharesPerPeriod")
                                .withKeySerde(stringSerde)
                                .withValueSerde(Serdes.Integer()))
                .toStream().peek((k,v)->LOG.info("key is {} value is {}", k, v))
                .to("transaction-count", Produced.with(windowedSerde,Serdes.Integer()));


        KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig);
        InteractiveQueryServer queryServer = new InteractiveQueryServer(kafkaStreams, hostInfo);
        StateRestoreHttpReporter restoreReporter = new StateRestoreHttpReporter(queryServer);

        queryServer.init();

        kafkaStreams.setGlobalStateRestoreListener(restoreReporter);

        kafkaStreams.setStateListener(((newState, oldState) -> {
            if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) {
                LOG.info("Setting the query server to ready");
                queryServer.setReady(true);
            } else if (newState != KafkaStreams.State.RUNNING) {
                LOG.info("State not RUNNING, disabling the query server");
                queryServer.setReady(false);
            }
        }));

        kafkaStreams.setUncaughtExceptionHandler((t, e) -> {
            LOG.error("Thread {} had a fatal error {}", t, e, e);
            shutdown(kafkaStreams, queryServer);
        });


        Runtime.getRuntime().addShutdownHook(new Thread(() -> {
            shutdown(kafkaStreams, queryServer);
        }));

        LOG.info("Stock Analysis KStream Interactive Query App Started");
        kafkaStreams.cleanUp();
        kafkaStreams.start();
    }
 
Example #30
Source File: WindowFinalResultTest.java    From kafka-tutorials with Apache License 2.0 4 votes vote down vote up
private TimeWindows makeFixedTimeWindow() {
    return TimeWindows.of(testWindowSize).advanceBy(testWindowSize).grace(testGracePeriodSize);
}