org.apache.kafka.streams.state.QueryableStoreTypes Java Examples

public AbstractKafkaRepository(String name, Serde<V> valueSerde, Function<V,String> keyFunction, KafkaAdminService kafkaAdminService, KafkaConfiguration kafkaConfiguration) {
    this.repositoryName = name + "-db";
    this.keyFunction = keyFunction;
    this.producer = KafkaUtils.kafkaProducer(kafkaConfiguration.getBootstrapServers(), StringSerializer.class, JsonNodeSerialializer.class);
    kafkaAdminService.createTopic(kafkaAdminService.buildTopicInfo(repositoryName,TopicConfig.CLEANUP_POLICY_COMPACT));

    Properties props = KafkaUtils.createKStreamProperties(repositoryName + "-stream"+ UUID.randomUUID().toString(), kafkaConfiguration.getBootstrapServers());
    StreamsBuilder builder = new StreamsBuilder();

    final GlobalKTable<String, V> globalKTable = builder.globalTable(repositoryName, materialize(valueSerde));

    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    producer.flush();
    keyValueStore = streams.store(getStoreName(), QueryableStoreTypes.keyValueStore());

    Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

}
 

@Get("/autocompleteTags")
@ProducesJson
public JsonNode getAutocompleteTags() {
  try {
    if (!storage.traceSearchEnabled) return MAPPER.createArrayNode();
    ReadOnlyKeyValueStore<String, Set<String>> autocompleteTagsStore =
        storage.getTraceStorageStream().store(AUTOCOMPLETE_TAGS_STORE_NAME,
            QueryableStoreTypes.keyValueStore());
    ArrayNode array = MAPPER.createArrayNode();
    try (KeyValueIterator<String, Set<String>> all = autocompleteTagsStore.all()) {
      all.forEachRemaining(keyValue -> array.add(keyValue.key));
    }
    return array;
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    throw e;
  }
}
 

/**
 * get Metrics for a machine
 * @param machine
 * @return 
 */
private Metrics getLocalMetrics(String machine) {
    LOGGER.log(Level.INFO, "Getting Metrics for machine {0}", machine);
    
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());

    localMetrics.add(source, machine, String.valueOf(averageStore.get(machine)));

    LOGGER.log(Level.INFO, "Metrics for machine {0} - {1}", new Object[]{machine, localMetrics});
    return localMetrics;
}
 

@Override
public void start() throws Exception {

    execute(Config.RANGE_KEY_VALUE_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> {

        RangeKeyValueQuery query = (RangeKeyValueQuery) abstractQuery;
        ReadOnlyKeyValueStore<Object, Object> kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore());
        MultiValuedKeyValueQueryResponse response;
        try (KeyValueIterator<Object, Object> result = kvStore.range(deserializeObject(keySerde, query.getFrom()), deserializeObject(keySerde, query.getTo()))) {
            if (result.hasNext()) {
                Map<String, String> results = new HashMap<>();
                while (result.hasNext()) {
                    KeyValue<Object, Object> kvEntry = result.next();
                    results.put(base64Encode(keySerde, kvEntry.key), base64Encode(valueSerde, kvEntry.value));
                }
                return new MultiValuedKeyValueQueryResponse(results);
            } else {
               return new MultiValuedKeyValueQueryResponse(Collections.emptyMap());
            }
        }
    } );


}
 

@Override
public void start() throws Exception {

    execute(Config.ALL_KEY_VALUE_QUERY_ADDRESS_PREFIX, (query, keySerde, valueSerde) -> {

        ReadOnlyKeyValueStore<Object, Object> kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore());
        MultiValuedKeyValueQueryResponse response;
        try (KeyValueIterator<Object, Object> result = kvStore.all()) {
            if (result.hasNext()) {
                Map<String, String> results = new HashMap<>();
                while (result.hasNext()) {
                    KeyValue<Object, Object> kvEntry = result.next();

                    results.put(base64Encode(keySerde, kvEntry.key), base64Encode(valueSerde, kvEntry.value));
                }
               return new MultiValuedKeyValueQueryResponse(results);
            } else {
               return new MultiValuedKeyValueQueryResponse(Collections.emptyMap());
            }
        }
    });

}
 

@Override
public void start() throws Exception {

    execute(Config.KEY_VALUE_QUERY_ADDRESS_PREFIX, (abstractQuery, keySerde, valueSerde) -> {
        KeyBasedQuery query = (KeyBasedQuery) abstractQuery;

        Object deserializedKey = deserializeObject(keySerde, query.getKey());
        ReadOnlyKeyValueStore<Object, Object> kvStore = streams.store(query.getStoreName(), QueryableStoreTypes.keyValueStore());
        Object result = kvStore.get(deserializedKey);
        if (result != null) {
            return new ScalarKeyValueQueryResponse(base64Encode(valueSerde, result));
        } else {
            throw new ScalarValueNotFoundException(String.format("Key %s not found in store %s", deserializedKey.toString(), abstractQuery.getStoreName()));
        }
    });

}
 

@Test
public void testStateStoreRetrievalRetry() {

	StreamsBuilderFactoryBean mock = Mockito.mock(StreamsBuilderFactoryBean.class);
	KafkaStreams mockKafkaStreams = Mockito.mock(KafkaStreams.class);
	Mockito.when(mock.getKafkaStreams()).thenReturn(mockKafkaStreams);
	KafkaStreamsRegistry kafkaStreamsRegistry = new KafkaStreamsRegistry();
	kafkaStreamsRegistry.registerKafkaStreams(mock);
	KafkaStreamsBinderConfigurationProperties binderConfigurationProperties =
			new KafkaStreamsBinderConfigurationProperties(new KafkaProperties());
	binderConfigurationProperties.getStateStoreRetry().setMaxAttempts(3);
	InteractiveQueryService interactiveQueryService = new InteractiveQueryService(kafkaStreamsRegistry,
			binderConfigurationProperties);

	QueryableStoreType<ReadOnlyKeyValueStore<Object, Object>> storeType = QueryableStoreTypes.keyValueStore();
	try {
		interactiveQueryService.getQueryableStore("foo", storeType);
	}
	catch (Exception ignored) {

	}

	Mockito.verify(mockKafkaStreams, times(3)).store("foo", storeType);
}
 

/**
 * Query local state store to extract metrics
 *
 * @return local Metrics
 */
private Metrics getLocalMetrics() {
    HostInfo thisInstance = GlobalAppState.getInstance().getHostPortInfo();
    KafkaStreams ks = GlobalAppState.getInstance().getKafkaStreams();

    String source = thisInstance.host() + ":" + thisInstance.port();
    Metrics localMetrics = new Metrics();

    ReadOnlyKeyValueStore<String, Double> averageStore = ks
            .store(storeName,
                    QueryableStoreTypes.<String, Double>keyValueStore());

    LOGGER.log(Level.INFO, "Entries in store {0}", averageStore.approximateNumEntries());
    KeyValueIterator<String, Double> storeIterator = averageStore.all();

    while (storeIterator.hasNext()) {
        KeyValue<String, Double> kv = storeIterator.next();
        localMetrics.add(source, kv.key, String.valueOf(kv.value));

    }
    LOGGER.log(Level.INFO, "Local store state {0}", localMetrics);
    return localMetrics;
}
 

@Get("/autocompleteTags/:key")
@ProducesJson
public JsonNode getAutocompleteValues(@Param("key") String key) {
  try {
    if (!storage.traceSearchEnabled) return MAPPER.createArrayNode();
    ReadOnlyKeyValueStore<String, Set<String>> autocompleteTagsStore =
        storage.getTraceStorageStream().store(AUTOCOMPLETE_TAGS_STORE_NAME,
            QueryableStoreTypes.keyValueStore());
    Set<String> values = autocompleteTagsStore.get(key);
    ArrayNode array = MAPPER.createArrayNode();
    if (values != null) values.forEach(array::add);
    return array;
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    throw e;
  }
}
 

@Get("/traceMany")
public AggregatedHttpResponse getTraces(@Param("traceIds") String traceIds) {
  try {
    if (!storage.traceByIdQueryEnabled) return AggregatedHttpResponse.of(HttpStatus.NOT_FOUND);
    ReadOnlyKeyValueStore<String, List<Span>> store = storage.getTraceStorageStream()
        .store(TRACES_STORE_NAME, QueryableStoreTypes.keyValueStore());
    List<List<Span>> result = new ArrayList<>();
    for (String traceId : traceIds.split(",", 1000)) {
      result.add(store.get(traceId));
    }
    return AggregatedHttpResponse.of(HttpStatus.OK, MediaType.JSON, writeTraces(result));
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    return AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE);
  }
}
 

@Get("/traces/:trace_id")
public AggregatedHttpResponse getTrace(@Param("trace_id") String traceId) {
  try {
    if (!storage.traceByIdQueryEnabled) return AggregatedHttpResponse.of(HttpStatus.NOT_FOUND);
    ReadOnlyKeyValueStore<String, List<Span>> store =
        storage.getTraceStorageStream()
            .store(TRACES_STORE_NAME, QueryableStoreTypes.keyValueStore());
    List<Span> spans = store.get(traceId);
    return AggregatedHttpResponse.of(
        HttpStatus.OK,
        MediaType.JSON,
        SpanBytesEncoder.JSON_V2.encodeList(spans));
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    return AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE);
  }
}
 

@Get("/serviceNames/:service_name/remoteServiceNames")
@ProducesJson
public JsonNode getRemoteServiceNames(@Param("service_name") String serviceName) {
  try {
    if (!storage.traceSearchEnabled) return MAPPER.createArrayNode();
    ReadOnlyKeyValueStore<String, Set<String>> store = storage.getTraceStorageStream()
        .store(REMOTE_SERVICE_NAMES_STORE_NAME, QueryableStoreTypes.keyValueStore());
    Set<String> names = store.get(serviceName);
    ArrayNode array = MAPPER.createArrayNode();
    if (names != null) names.forEach(array::add);
    return (array);
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    throw e;
  }
}
 

@Get("/serviceNames/:service_name/spanNames")
@ProducesJson
public JsonNode getSpanNames(@Param("service_name") String serviceName) {
  try {
    if (!storage.traceSearchEnabled) return MAPPER.createArrayNode();
    ReadOnlyKeyValueStore<String, Set<String>> store = storage.getTraceStorageStream()
        .store(SPAN_NAMES_STORE_NAME, QueryableStoreTypes.keyValueStore());
    Set<String> names = store.get(serviceName);
    ArrayNode array = MAPPER.createArrayNode();
    if (names != null) names.forEach(array::add);
    return array;
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    throw e;
  }
}
 

@Get("/serviceNames")
@ProducesJson
public JsonNode getServiceNames() {
  try {
    if (!storage.traceSearchEnabled) return MAPPER.createArrayNode();
    ReadOnlyKeyValueStore<String, String> store = storage.getTraceStorageStream()
        .store(SERVICE_NAMES_STORE_NAME, QueryableStoreTypes.keyValueStore());
    ArrayNode array = MAPPER.createArrayNode();
    try (KeyValueIterator<String, String> all = store.all()) {
      all.forEachRemaining(keyValue -> array.add(keyValue.value));
    }
    return array;
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    throw e;
  }
}
 

public Long getProductStock(Integer id) {
	ReadOnlyKeyValueStore<Object, Object> keyValueStore = interactiveQueryService
			.getQueryableStore("prod-id-count-store",
					QueryableStoreTypes.keyValueStore());

	return (Long) keyValueStore.get(id);
}
 

private ReadOnlyKeyValueStore<Integer, Long> getCountstore() {
    while (true) {
        try {
            return streams.store("countstore", QueryableStoreTypes.keyValueStore());
        } catch (InvalidStateStoreException e) {
            // ignore, store not ready yet
        }
    }
}
 

@GetMapping("/count/{status}")
public long getVehicleCoutnForStatus(@PathVariable("status") String status) {
	// Get the state-store
	ReadOnlyKeyValueStore<String, Long> keyValueStore= kStreamBuilderFactoryBean.getKafkaStreams()
													.store("statusCount", QueryableStoreTypes.keyValueStore());
	return keyValueStore.get(status);	//get the count for the key viz. Offline/Online
}
 

public static <K, V> NavigableMap<K, V> mapFromStore(KafkaStreams streams, String storeName) {
    final ReadOnlyKeyValueStore<K, V> store = streams.store(
        storeName, QueryableStoreTypes.keyValueStore());

    try (final KeyValueIterator<K, V> all = store.all()) {
        NavigableMap<K, V> result = new TreeMap<>();
        while (all.hasNext()) {
            KeyValue<K, V> next = all.next();
            result.put(next.key, next.value);
        }
        return result;
    }
}
 

public static <K, V> List<KeyValue<K, V>> listFromStore(KafkaStreams streams, String storeName) {
    final ReadOnlyKeyValueStore<K, V> store = streams.store(
        storeName, QueryableStoreTypes.keyValueStore());

    try (final KeyValueIterator<K, V> all = store.all()) {
        List<KeyValue<K, V>> result = new ArrayList<>();
        while (all.hasNext()) {
            result.add(all.next());
        }
        return result;
    }
}
 

private String readKeyFromLocal(String searchKey)
    throws JsonProcessingException {
    ReadOnlyKeyValueStore<String, DepartmentAggregate> departmentStore =
        streams.store(
            AppConfigs.aggStateStoreName,
            QueryableStoreTypes.keyValueStore()
        );
    DepartmentAggregate result = departmentStore.get(searchKey);
    return (result == null) ? NO_RESULTS
        : objectMapper.writeValueAsString(result);
}
 

@Scheduled(fixedRate = 30000, initialDelay = 5000)
public void printVersionCounts() {
    if (keyValueStore == null) {
        keyValueStore = queryService.getQueryableStore(STORE_NAME, QueryableStoreTypes.keyValueStore());
    }

    logger.info("Count for v1 is=" + keyValueStore.get("v1"));
    logger.info("Count for v2 is=" + keyValueStore.get("v2"));
}
 

@Scheduled(fixedRate = 30000, initialDelay = 5000)
public void printVersionCounts() {
	if (keyValueStore == null) {
		keyValueStore = queryService.getQueryableStore(STORE_NAME, QueryableStoreTypes.keyValueStore());
	}

	logger.info("Count for v1 is=" + keyValueStore.get("v1"));
	logger.info("Count for v2 is=" + keyValueStore.get("v2"));
}
 

@RequestMapping("/events")
public String events() {

	final ReadOnlyKeyValueStore<String, String> topFiveStore =
			interactiveQueryService.getQueryableStore("test-events-snapshots", QueryableStoreTypes.<String, String>keyValueStore());
	return topFiveStore.get("12345");
}
 

@Scheduled(fixedRate = 30000, initialDelay = 5000)
public void printProductCounts() {
	if (keyValueStore == null) {
		keyValueStore = queryService.getQueryableStore(STORE_NAME, QueryableStoreTypes.keyValueStore());
	}

	for (Integer id : productIds()) {
		System.out.println("Product ID: " + id + " Count: " + keyValueStore.get(id));
	}
}
 

private ReadOnlyKeyValueStore<Integer, Aggregation> getWeatherStationStore() {
    while (true) {
        try {
            return streams.store(TopologyProducer.WEATHER_STATIONS_STORE, QueryableStoreTypes.keyValueStore());
        } catch (InvalidStateStoreException e) {
            // ignore, store not ready yet
        }
    }
}
 

@Override
protected ExtReadOnlyKeyValueStore<K, V> localService(String storeName, KafkaStreams streams) {
    QueryableStoreType<ReadOnlyKeyValueStore<K, V>> queryableStoreType = QueryableStoreTypes.keyValueStore();
    StoreQueryParameters<ReadOnlyKeyValueStore<K, V>> sqp = StoreQueryParameters.fromNameAndType(storeName, queryableStoreType);
    ReadOnlyKeyValueStore<K, V> delegate = streams.store(sqp);
    return new ExtReadOnlyKeyValueStoreImpl<>(delegate, filterPredicate);
}
 

@Get("/dependencies")
public AggregatedHttpResponse getDependencies(
    @Param("endTs") long endTs,
    @Param("lookback") long lookback
) {
  try {
    if (!storage.dependencyQueryEnabled) return AggregatedHttpResponse.of(HttpStatus.NOT_FOUND);
    ReadOnlyWindowStore<Long, DependencyLink> store =
        storage.getDependencyStorageStream()
            .store(DEPENDENCIES_STORE_NAME, QueryableStoreTypes.windowStore());
    List<DependencyLink> links = new ArrayList<>();
    Instant from = Instant.ofEpochMilli(endTs - lookback);
    Instant to = Instant.ofEpochMilli(endTs);
    try (KeyValueIterator<Windowed<Long>, DependencyLink> iterator = store.fetchAll(from, to)) {
      iterator.forEachRemaining(keyValue -> links.add(keyValue.value));
    }
    List<DependencyLink> mergedLinks = DependencyLinker.merge(links);
    LOG.debug("Dependencies found from={}-to={}: {}", from, to, mergedLinks.size());
    return AggregatedHttpResponse.of(
        HttpStatus.OK,
        MediaType.JSON,
        DependencyLinkBytesEncoder.JSON_V1.encodeList(mergedLinks));
  } catch (InvalidStateStoreException e) {
    LOG.debug("State store is not ready", e);
    return AggregatedHttpResponse.of(HttpStatus.SERVICE_UNAVAILABLE);
  }
}
 

private String getKeyValuesAsJson(String store) {
    ReadOnlyKeyValueStore<String, Long> readOnlyKeyValueStore = kafkaStreams.store(store, QueryableStoreTypes.keyValueStore());
    List<KeyValue<String, Long>> keyValues = new ArrayList<>();
    try (KeyValueIterator<String, Long> iterator = readOnlyKeyValueStore.all()) {
        while (iterator.hasNext()) {
            keyValues.add(iterator.next());
        }
    }
    return gson.toJson(keyValues);
}
 

private String fetchFromSessionStore(Map<String, String> params) {
    String store = params.get(STORE_PARAM);
    String key = params.get(KEY_PARAM);

    HostInfo storeHostInfo = getHostInfo(store, key);

    if (storeHostInfo.host().equals("unknown")) {
        return STORES_NOT_ACCESSIBLE;
    }

    if (dataNotLocal(storeHostInfo)) {
        LOG.info("{} located in state store on another instance !!!!", key);
        return fetchRemote(storeHostInfo, "session", params);
    }

    ReadOnlySessionStore<String, CustomerTransactions> readOnlySessionStore = kafkaStreams.store(store, QueryableStoreTypes.sessionStore());
    List<String> results = new ArrayList<>();
    List<KeyValue<String, List<String>>> sessionResults = new ArrayList<>();
    try (KeyValueIterator<Windowed<String>, CustomerTransactions> iterator = readOnlySessionStore.fetch(key)) {
        while (iterator.hasNext()) {
            KeyValue<Windowed<String>, CustomerTransactions> windowed = iterator.next();
            CustomerTransactions transactions = windowed.value;
            LocalDateTime startSession = getLocalDateTime(windowed.key.window().start());
            LocalDateTime endSession = getLocalDateTime(windowed.key.window().end());
            transactions.setSessionInfo(String.format("Session Window{start=%s, end=%s}", startSession.toLocalTime().toString(), endSession.toLocalTime().toString()));
            results.add(transactions.toString());
        }
        sessionResults.add(new KeyValue<>(key, results));
    }

    return gson.toJson(sessionResults);
}
 

@SuppressWarnings({"unchecked", "rawtypes"})
private <K, V> ExtReadOnlyKeyValueStore<K, V> keyValueStore(String storeName) {
    return (ExtReadOnlyKeyValueStore<K, V>)
        keyValueStores.computeIfAbsent(
                storeName,
                sn -> {
                    QueryableStoreType<ReadOnlyKeyValueStore<K, V>> queryableStoreType = QueryableStoreTypes.keyValueStore();
                    StoreQueryParameters<ReadOnlyKeyValueStore<K, V>> sqp = StoreQueryParameters.fromNameAndType(storeName, queryableStoreType);
                    return new ExtReadOnlyKeyValueStoreImpl(
                            streams.store(sqp),
                            filterPredicate
                    );
                }
        );
}