org.apache.kafka.streams.kstream.TimeWindows Java Examples
The following examples show how to use
org.apache.kafka.streams.kstream.TimeWindows.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: UserClicksPerMinute.java From fluent-kafka-streams-tests with MIT License | 8 votes |
public Topology getTopology() { final StreamsBuilder builder = new StreamsBuilder(); final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.inputTopic); final KTable<Windowed<Integer>, Long> counts = clickEvents .groupByKey() .windowedBy(TimeWindows.of(Duration.ofMinutes(1))) .count(); counts.toStream() .map((key, value) -> KeyValue.pair( key.key(), new ClickOutput(key.key(), value, key.window().start()))) .to(this.outputTopic, Produced.with(Serdes.Integer(), new JsonSerde<>(ClickOutput.class))); return builder.build(); }
Example #2
Source File: KafkaStreamsBinderWordCountIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 6 votes |
@StreamListener @SendTo("output") public KStream<?, WordCount> process( @Input("input") KStream<Object, String> input) { return input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #3
Source File: WordCountMultipleBranchesIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 6 votes |
@StreamListener("input") @SendTo({ "output1", "output2", "output3" }) @SuppressWarnings("unchecked") public KStream<?, WordCount>[] process(KStream<Object, String> input) { Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english"); Predicate<Object, WordCount> isFrench = (k, v) -> v.word.equals("french"); Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish"); return input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .groupBy((key, value) -> value).windowedBy(TimeWindows.of(Duration.ofSeconds(5))) .count(Materialized.as("WordCounts-multi")).toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) .branch(isEnglish, isFrench, isSpanish); }
Example #4
Source File: KafkaStreamsBinderWordCountBranchesFunctionTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 6 votes |
@Bean @SuppressWarnings("unchecked") public Function<KStream<Object, String>, KStream<?, WordCount>[]> process() { Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english"); Predicate<Object, WordCount> isFrench = (k, v) -> v.word.equals("french"); Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish"); return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .groupBy((key, value) -> value) .windowedBy(TimeWindows.of(5000)) .count(Materialized.as("WordCounts-branch")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) .branch(isEnglish, isFrench, isSpanish); }
Example #5
Source File: MainVerticle.java From kiqr with Apache License 2.0 | 6 votes |
@Override public void start(Future<Void> startFuture) throws Exception { Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "kiqr"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.StringSerde.class); props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.LongSerde.class); props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, "0"); KStreamBuilder builder = new KStreamBuilder(); KTable<String, Long> table = builder.table(Serdes.String(), Serdes.Long(), "visits", "visitStore"); KTable<Windowed<String>, Long> windowedCount = table.toStream().groupByKey().count(TimeWindows.of(60), "visitCount"); vertx.deployVerticle(RestKiqrServerVerticle.Builder.serverBuilder(builder, props).withPort(2901).build(), res -> { if (res.succeeded()) { startFuture.complete(); } else { startFuture.fail(res.cause()); } }); }
Example #6
Source File: KafkaStreamsBranchingSample.java From spring-cloud-stream-samples with Apache License 2.0 | 6 votes |
@Bean @SuppressWarnings("unchecked") public Function<KStream<Object, String>, KStream<?, WordCount>[]> process() { Predicate<Object, WordCount> isEnglish = (k, v) -> v.word.equals("english"); Predicate<Object, WordCount> isFrench = (k, v) -> v.word.equals("french"); Predicate<Object, WordCount> isSpanish = (k, v) -> v.word.equals("spanish"); return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .groupBy((key, value) -> value) .windowedBy(TimeWindows.of(Duration.ofSeconds(6))) .count(Materialized.as("WordCounts-1")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) .branch(isEnglish, isFrench, isSpanish); }
Example #7
Source File: HoppingWindowExpressionTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldCreateHoppingWindowAggregate() { final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class); final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class); final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class); final HoppingWindowExpression windowExpression = new HoppingWindowExpression(10, TimeUnit.SECONDS, 4, TimeUnit.MILLISECONDS); final Initializer initializer = () -> 0; final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store"); EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L).advanceBy(4L))).andReturn(windowedKStream); EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null); EasyMock.replay(stream, windowedKStream); windowExpression.applyAggregate(stream, initializer, aggregator, store); EasyMock.verify(stream, windowedKStream); }
Example #8
Source File: TumblingWindowExpressionTest.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 6 votes |
@Test public void shouldCreateTumblingWindowAggregate() { final KGroupedStream stream = EasyMock.createNiceMock(KGroupedStream.class); final TimeWindowedKStream windowedKStream = EasyMock.createNiceMock(TimeWindowedKStream.class); final UdafAggregator aggregator = EasyMock.createNiceMock(UdafAggregator.class); final TumblingWindowExpression windowExpression = new TumblingWindowExpression(10, TimeUnit.SECONDS); final Initializer initializer = () -> 0; final Materialized<String, GenericRow, WindowStore<Bytes, byte[]>> store = Materialized.as("store"); EasyMock.expect(stream.windowedBy(TimeWindows.of(10000L))).andReturn(windowedKStream); EasyMock.expect(windowedKStream.aggregate(same(initializer), same(aggregator), same(store))).andReturn(null); EasyMock.replay(stream, windowedKStream); windowExpression.applyAggregate(stream, initializer, aggregator, store); EasyMock.verify(stream, windowedKStream); }
Example #9
Source File: KafkaStreamsProductTrackerApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, Product>, KStream<Integer, ProductStatus>> process() { return input -> input .filter((key, product) -> productIds().contains(product.getId())) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(TimeWindows.of(Duration.ofSeconds(60))) .count(Materialized.as("product-counts")) .toStream() .map((key, value) -> new KeyValue<>(key.key().id, new ProductStatus(key.key().id, value, Instant.ofEpochMilli(key.window().start()).atZone(ZoneId.systemDefault()).toLocalTime(), Instant.ofEpochMilli(key.window().end()).atZone(ZoneId.systemDefault()).toLocalTime()))); }
Example #10
Source File: KafkaStreamsDlqSample.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, String>,KStream<?, WordCount>> process() { return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(5000)) .count(Materialized.as("WordCounts-1")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #11
Source File: KafkaStreamsWordCountApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Bytes, String>, KStream<Bytes, WordCount>> process() { return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofMillis(WINDOW_SIZE_MS))) .count(Materialized.as("WordCounts-1")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #12
Source File: KafkaStreamsWordCountApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, String>, KStream<?, WordCount>> process() { return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofSeconds(60))) .count(Materialized.as("WordCounts-1")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #13
Source File: KafkaStreamsBinderWordCountFunctionTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, String>, KStream<String, WordCount>> process() { return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(5000)) .count(Materialized.as("foo-WordCounts")) .toStream() .map((key, value) -> new KeyValue<>(key.key(), new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #14
Source File: KafkaStreamsWordCountApplication.java From spring-cloud-stream-samples with Apache License 2.0 | 5 votes |
@Bean public Function<KStream<Object, String>, KStream<?, WordCount>> process() { return input -> input .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofSeconds(20))) .count(Materialized.as("WordCounts-1")) .toStream() .map((key, value) -> new KeyValue<>(null, new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))); }
Example #15
Source File: KafkaStreamsNativeEncodingDecodingTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@StreamListener("input") @SendTo("output") public KStream<?, String> process(KStream<Object, String> input) { return input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts-x")) .toStream().map((key, value) -> new KeyValue<>(null, "Count for " + key.key() + " : " + value)); }
Example #16
Source File: StocksKafkaStreamsDriver.java From kafka-streams with Apache License 2.0 | 5 votes |
public static void main(String[] args) { StreamsConfig streamingConfig = new StreamsConfig(getProperties()); JsonSerializer<StockTransactionCollector> stockTransactionsSerializer = new JsonSerializer<>(); JsonDeserializer<StockTransactionCollector> stockTransactionsDeserializer = new JsonDeserializer<>(StockTransactionCollector.class); JsonDeserializer<StockTransaction> stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class); JsonSerializer<StockTransaction> stockTxnJsonSerializer = new JsonSerializer<>(); Serde<StockTransaction> transactionSerde = Serdes.serdeFrom(stockTxnJsonSerializer,stockTxnDeserializer); StringSerializer stringSerializer = new StringSerializer(); StringDeserializer stringDeserializer = new StringDeserializer(); Serde<String> stringSerde = Serdes.serdeFrom(stringSerializer,stringDeserializer); Serde<StockTransactionCollector> collectorSerde = Serdes.serdeFrom(stockTransactionsSerializer,stockTransactionsDeserializer); WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerializer); WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringDeserializer); Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer,windowedDeserializer); KStreamBuilder kStreamBuilder = new KStreamBuilder(); KStream<String,StockTransaction> transactionKStream = kStreamBuilder.stream(stringSerde,transactionSerde,"stocks"); transactionKStream.map((k,v)-> new KeyValue<>(v.getSymbol(),v)) .through(stringSerde, transactionSerde,"stocks-out") .groupBy((k,v) -> k, stringSerde, transactionSerde) .aggregate(StockTransactionCollector::new, (k, v, stockTransactionCollector) -> stockTransactionCollector.add(v), TimeWindows.of(10000), collectorSerde, "stock-summaries") .to(windowedSerde,collectorSerde,"transaction-summary"); System.out.println("Starting StockStreams Example"); KafkaStreams kafkaStreams = new KafkaStreams(kStreamBuilder,streamingConfig); kafkaStreams.start(); System.out.println("Now started StockStreams Example"); }
Example #17
Source File: OutboundValueNullSkippedConversionTest.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@StreamListener @SendTo("output") public KStream<?, KafkaStreamsBinderWordCountIntegrationTests.WordCount> process( @Input("input") KStream<Object, String> input) { return input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(Duration.ofSeconds(5))).count(Materialized.as("foo-WordCounts")) .toStream() .map((key, value) -> new KeyValue<>(null, null)); }
Example #18
Source File: DeserializationErrorHandlerByKafkaTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@StreamListener("input") @SendTo("output") public KStream<?, String> process(KStream<Object, String> input) { return input .flatMapValues( value -> Arrays.asList(value.toLowerCase().split("\\W+"))) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Serialized.with(Serdes.String(), Serdes.String())) .windowedBy(TimeWindows.of(5000)).count(Materialized.as("foo-WordCounts-x")) .toStream().map((key, value) -> new KeyValue<>(null, "Count for " + key.key() + " : " + value)); }
Example #19
Source File: ErrorEventsPerMinute.java From fluent-kafka-streams-tests with MIT License | 5 votes |
public Topology getTopology() { final StreamsBuilder builder = new StreamsBuilder(); // Click Events final KStream<Integer, ClickEvent> clickEvents = builder.stream(this.clickInputTopic, Consumed.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class))); final KTable<Windowed<Integer>, Long> counts = clickEvents .selectKey(((key, value) -> value.getStatus())) .filter(((key, value) -> key >= 400)) .groupByKey(Grouped.with(Serdes.Integer(), new JsonSerde<>(ClickEvent.class))) .windowedBy(TimeWindows.of(Duration.ofMinutes(1))) // 1 Minute in ms .count(); // Status codes final KTable<Integer, StatusCode> statusCodes = builder.table(this.statusInputTopic, Consumed.with(Serdes.Integer(), new JsonSerde<>(StatusCode.class))); // Join final KStream<Integer, ErrorOutput> errors = counts.toStream() .map((key, value) -> KeyValue.pair( key.key(), new ErrorOutput(key.key(), value, key.window().start(), null /*empty definition*/))) .join(statusCodes, (countRecord, code) -> new ErrorOutput( countRecord.getStatusCode(), countRecord.getCount(), countRecord.getTime(), code.getDefinition()), Joined.valueSerde(new JsonSerde<>(ErrorOutput.class))); errors.to(this.errorOutputTopic); // Send alert if more than 5x a certain error code per minute errors.filter((key, errorOutput) -> errorOutput.getCount() > 5L).to(this.alertTopic); return builder.build(); }
Example #20
Source File: DeserializtionErrorHandlerByBinderTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@StreamListener("input") @SendTo("output") public KStream<Integer, Long> process(KStream<Object, Product> input) { return input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Grouped.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(TimeWindows.of(5000)) .count(Materialized.as("id-count-store-x")).toStream() .map((key, value) -> new KeyValue<>(key.key().id, value)); }
Example #21
Source File: KafkastreamsBinderPojoInputStringOutputIntegrationTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@StreamListener("input") @SendTo("output") public KStream<Integer, String> process(KStream<Object, Product> input) { return input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Serialized.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(TimeWindows.of(5000)) .count(Materialized.as("id-count-store")).toStream() .map((key, value) -> new KeyValue<>(key.key().id, "Count for product with ID 123: " + value)); }
Example #22
Source File: KafkaStreamsBinderPojoInputAndPrimitiveTypeOutputTests.java From spring-cloud-stream-binder-kafka with Apache License 2.0 | 5 votes |
@StreamListener("input") @SendTo("output") public KStream<Integer, Long> process(KStream<Object, Product> input) { return input.filter((key, product) -> product.getId() == 123) .map((key, value) -> new KeyValue<>(value, value)) .groupByKey(Serialized.with(new JsonSerde<>(Product.class), new JsonSerde<>(Product.class))) .windowedBy(TimeWindows.of(5000)) .count(Materialized.as("id-count-store-x")).toStream() .map((key, value) -> { return new KeyValue<>(key.key().id, value); }); }
Example #23
Source File: KafkaStreamingMain.java From kafka-streams-api-websockets with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { Properties props = new Properties(); props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streaming-example"); props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); props.put(StreamsConfig.COMMIT_INTERVAL_MS_CONFIG, 1500); // To get data produced before process started // props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); // props.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); KStreamBuilder builder = new KStreamBuilder(); KStream<String, String> source = builder.stream("data-in"); KStream<String, String> stats = source.groupByKey() .aggregate(KafkaStreamingStatistics::new, (k, v, clusterstats) -> clusterstats.add(v), TimeWindows.of(60000).advanceBy(10000), Serdes.serdeFrom(new MySerde(), new MySerde()), "data-store") .toStream((key, value) -> key.key().toString() + " " + key.window().start()) .mapValues((job) -> job.computeAvgTime().toString()); stats.to(Serdes.String(), Serdes.String(), "data-out"); KafkaStreams streams = new KafkaStreams(builder, props); streams.cleanUp(); streams.start(); Runtime.getRuntime().addShutdownHook(new Thread(streams::close)); }
Example #24
Source File: SummaryBulkAggregation.java From kafka-graphs with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public KTable<Windowed<Short>, T> run(final KStream<Edge<K>, EV> edgeStream) { //For parallel window support we key the edge stream by partition and apply a parallel fold per partition. //Finally, we merge all locally combined results into our final graph aggregation property. KTable<Windowed<Short>, S> partialAgg = edgeStream .groupByKey(Grouped.with(new KryoSerde<>(), new KryoSerde<>())) .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis))) .aggregate(this::initialValue, new PartialAgg<>(updateFun())) .toStream() .groupBy((k, v) -> GLOBAL_KEY) .windowedBy(TimeWindows.of(Duration.ofMillis(timeMillis))) .reduce(combineFun()) .mapValues(aggregator(edgeStream), Materialized.<Windowed<Short>, S, KeyValueStore<Bytes, byte[]>> as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>())); if (transform() != null) { return partialAgg.mapValues( transform(), Materialized.<Windowed<Short>, T, KeyValueStore<Bytes, byte[]>> as(KGraph.generateStoreName()).withKeySerde(new KryoSerde<>()).withValueSerde(new KryoSerde<>()) ); } return (KTable<Windowed<Short>, T>) partialAgg; }
Example #25
Source File: HoppingWindowExpression.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public KTable applyAggregate( KGroupedStream groupedStream, Initializer initializer, UdafAggregator aggregator, Materialized<String, GenericRow, ?> materialized ) { return groupedStream.windowedBy( TimeWindows.of(sizeUnit.toMillis(size)) .advanceBy(advanceByUnit.toMillis(advanceBy)) ).aggregate(initializer, aggregator, materialized); }
Example #26
Source File: TumblingWindowExpression.java From ksql-fork-with-deep-learning-function with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public KTable applyAggregate(final KGroupedStream groupedStream, final Initializer initializer, final UdafAggregator aggregator, final Materialized<String, GenericRow, ?> materialized) { return groupedStream.windowedBy(TimeWindows.of(sizeUnit.toMillis(size))) .aggregate(initializer, aggregator, materialized); }
Example #27
Source File: SpecificClientIntegrationITCase.java From kiqr with Apache License 2.0 | 4 votes |
@BeforeClass public static void produceMessages() throws Exception{ Properties producerProps = new Properties(); producerProps.put("bootstrap.servers", KAFKA_HOST + ":" + KAFKA_PORT); producerProps.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); producerProps.put("value.serializer", "org.apache.kafka.common.serialization.LongSerializer"); producerProps.put("linger.ms", 0); try(KafkaProducer<String, Long> producer = new KafkaProducer<>(producerProps)){ producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key1", 1L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100L, "key1", 2L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100000L, "key1", 3L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key2", 4L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100000L, "key2", 5L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100001L, "key2", 6L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key3", 7L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 50000L, "key3", 8L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 100001L, "key3", 9L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 0L, "key4", 10L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 1L, "key4", 11L)); producer.send(new ProducerRecord<String, Long>(TOPIC, 0, 2L, "key4", 12L)); } CountDownLatch cdl = new CountDownLatch(12); Properties consumerProps = new Properties(); consumerProps.put("bootstrap.servers", KAFKA_HOST + ":" + KAFKA_PORT); consumerProps.put("group.id", UUID.randomUUID().toString()); consumerProps.put("enable.auto.commit", "true"); consumerProps.put("auto.offset.reset", "earliest"); consumerProps.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); consumerProps.put("value.deserializer", "org.apache.kafka.common.serialization.LongDeserializer"); Runnable consumerRunnable = () -> { KafkaConsumer<String, Long> consumer = new KafkaConsumer<>(consumerProps); consumer.subscribe(Collections.singleton(TOPIC)); int tryCount = 10; while(true){ ConsumerRecords<String, Long> records = consumer.poll(500); records.forEach(rec -> cdl.countDown()); tryCount--; if(cdl.getCount() == 0){ consumer.close(); return; } else if(tryCount == 0){ throw new RuntimeException("times up"); } } }; consumerRunnable.run(); cdl.await(10000, TimeUnit.MILLISECONDS); KStreamBuilder builder = new KStreamBuilder(); KTable<String, Long> kv = builder.table(Serdes.String(), Serdes.Long(), TOPIC, "kv"); kv.toStream().groupByKey().count(TimeWindows.of(10000L), "window"); Properties streamProps = new Properties(); streamProps.put(StreamsConfig.APPLICATION_ID_CONFIG, UUID.randomUUID().toString()); streamProps.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, KAFKA_HOST + ":" + KAFKA_PORT); streamProps.put(StreamsConfig.CACHE_MAX_BYTES_BUFFERING_CONFIG, 0); streamProps.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass().getName()); streamProps.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Long().getClass().getName()); CountDownLatch streamCdl = new CountDownLatch(2); RestKiqrServerVerticle.Builder verticleBuilder = RestKiqrServerVerticle.Builder.serverBuilder(builder, streamProps); RuntimeVerticle.Builder builder1 = verticleBuilder.withPort(44321).withStateListener((newState, oldState) -> { if (newState == KafkaStreams.State.RUNNING) streamCdl.countDown(); System.out.println(oldState + " - " + newState); }); AbstractVerticle verticle = verticleBuilder.build(); CountDownLatch verticleCdl = new CountDownLatch(1); VERTX.deployVerticle(verticle, handler -> { verticleCdl.countDown(); }); streamCdl.await(100000, TimeUnit.MILLISECONDS); verticleCdl.await(100000, TimeUnit.MILLISECONDS); }
Example #28
Source File: StatKStreamBuilderSupplier.java From DBus with Apache License 2.0 | 4 votes |
@Override public KStreamBuilder get() { KStreamBuilder builder = new KStreamBuilder(); KStream<String, String> stream = builder.stream((String[]) sources.toArray()); KStream<HBKeySupplier.HBKey, String>[] streams = stream.filter((k, v) -> StringUtils.startsWith(v, "data_increment_heartbeat")) .selectKey((k, v) -> new HBKeySupplier(k).get()) .filter((k, v) -> k.isNormalFormat) .flatMapValues(v -> Arrays.asList("stat", "monitor")) .branch((k, v) -> StringUtils.equalsIgnoreCase("stat", v), (k, v) -> StringUtils.equalsIgnoreCase("monitor", v)); streams[0].transform(StatTransformer::new).to(sink); KStream<String, PacketVo> monitor = streams[1].filterNot((k, v) -> !StringUtils.equalsIgnoreCase("abort", k.getStatus())) .map((k, v) -> { StringJoiner joiner = new StringJoiner("/"); joiner.add(HeartBeatConfigContainer.getInstance().getHbConf().getMonitorPath()) .add(k.getDs()) .add(StringUtils.equalsIgnoreCase(DbusDatasourceType.DB2.name(), k.getDbType()) ? StringUtils.upperCase(k.getSchema()) : k.getSchema()) .add(k.getTable()).add(String.valueOf(k.getPartition())); String node = joiner.toString(); PacketVo packet = new PacketVo(); packet.setNode(node); packet.setType("checkpoint"); packet.setTime(k.getCheckpointMs()); packet.setTxTime(k.getTxTimeMs()); return new KeyValue(node, packet); }); // 需要先进行shuff把key相同的分配到partition号 monitor.through("monitor-repartition") .reduceByKey((agg, v) -> v.getTime() > agg.getTime() ? v : agg, TimeWindows.of("monitor", 2 * 60 * 1000)) .toStream() .map((k, v) -> new KeyValue<>(k.key(), v)) .process(new MonitorProcessorSupplier(), "zkInfo"); return builder; }
Example #29
Source File: StockPerformanceInteractiveQueryApplication.java From kafka-streams-in-action with Apache License 2.0 | 4 votes |
public static void main(String[] args) { if(args.length < 2){ LOG.error("Need to specify host, port"); System.exit(1); } String host = args[0]; int port = Integer.parseInt(args[1]); final HostInfo hostInfo = new HostInfo(host, port); Properties properties = getProperties(); properties.put(StreamsConfig.APPLICATION_SERVER_CONFIG, host+":"+port); StreamsConfig streamsConfig = new StreamsConfig(properties); Serde<String> stringSerde = Serdes.String(); Serde<Long> longSerde = Serdes.Long(); Serde<StockTransaction> stockTransactionSerde = StreamsSerdes.StockTransactionSerde(); WindowedSerializer<String> windowedSerializer = new WindowedSerializer<>(stringSerde.serializer()); WindowedDeserializer<String> windowedDeserializer = new WindowedDeserializer<>(stringSerde.deserializer()); Serde<Windowed<String>> windowedSerde = Serdes.serdeFrom(windowedSerializer, windowedDeserializer); Serde<CustomerTransactions> customerTransactionsSerde = StreamsSerdes.CustomerTransactionsSerde(); Aggregator<String, StockTransaction, Integer> sharesAggregator = (k, v, i) -> v.getShares() + i; StreamsBuilder builder = new StreamsBuilder(); // data is already coming in keyed KStream<String, StockTransaction> stockTransactionKStream = builder.stream(MockDataProducer.STOCK_TRANSACTIONS_TOPIC, Consumed.with(stringSerde, stockTransactionSerde) .withOffsetResetPolicy(Topology.AutoOffsetReset.LATEST)); stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getSector(), v)) .groupByKey(Serialized.with(stringSerde, stockTransactionSerde)) .count(Materialized.as("TransactionsBySector")) .toStream() .peek((k,v) -> LOG.info("Transaction count for {} {}", k, v)) .to("sector-transaction-counts", Produced.with(stringSerde, longSerde)); stockTransactionKStream.map((k,v) -> KeyValue.pair(v.getCustomerId(), v)) .groupByKey(Serialized.with(stringSerde, stockTransactionSerde)) .windowedBy(SessionWindows.with(TimeUnit.MINUTES.toMillis(60)).until(TimeUnit.MINUTES.toMillis(120))) .aggregate(CustomerTransactions::new,(k, v, ct) -> ct.update(v), (k, ct, other)-> ct.merge(other), Materialized.<String, CustomerTransactions, SessionStore<Bytes, byte[]>>as("CustomerPurchaseSessions") .withKeySerde(stringSerde).withValueSerde(customerTransactionsSerde)) .toStream() .peek((k,v) -> LOG.info("Session info for {} {}", k, v)) .to("session-transactions", Produced.with(windowedSerde, customerTransactionsSerde)); stockTransactionKStream.groupByKey(Serialized.with(stringSerde, stockTransactionSerde)) .windowedBy(TimeWindows.of(10000)) .aggregate(() -> 0, sharesAggregator, Materialized.<String, Integer, WindowStore<Bytes, byte[]>>as("NumberSharesPerPeriod") .withKeySerde(stringSerde) .withValueSerde(Serdes.Integer())) .toStream().peek((k,v)->LOG.info("key is {} value is {}", k, v)) .to("transaction-count", Produced.with(windowedSerde,Serdes.Integer())); KafkaStreams kafkaStreams = new KafkaStreams(builder.build(), streamsConfig); InteractiveQueryServer queryServer = new InteractiveQueryServer(kafkaStreams, hostInfo); StateRestoreHttpReporter restoreReporter = new StateRestoreHttpReporter(queryServer); queryServer.init(); kafkaStreams.setGlobalStateRestoreListener(restoreReporter); kafkaStreams.setStateListener(((newState, oldState) -> { if (newState == KafkaStreams.State.RUNNING && oldState == KafkaStreams.State.REBALANCING) { LOG.info("Setting the query server to ready"); queryServer.setReady(true); } else if (newState != KafkaStreams.State.RUNNING) { LOG.info("State not RUNNING, disabling the query server"); queryServer.setReady(false); } })); kafkaStreams.setUncaughtExceptionHandler((t, e) -> { LOG.error("Thread {} had a fatal error {}", t, e, e); shutdown(kafkaStreams, queryServer); }); Runtime.getRuntime().addShutdownHook(new Thread(() -> { shutdown(kafkaStreams, queryServer); })); LOG.info("Stock Analysis KStream Interactive Query App Started"); kafkaStreams.cleanUp(); kafkaStreams.start(); }
Example #30
Source File: WindowFinalResultTest.java From kafka-tutorials with Apache License 2.0 | 4 votes |
private TimeWindows makeFixedTimeWindow() { return TimeWindows.of(testWindowSize).advanceBy(testWindowSize).grace(testGracePeriodSize); }