Java Code Examples for org.apache.kafka.streams.processor.ProcessorContext#getStateStore()

The following examples show how to use org.apache.kafka.streams.processor.ProcessorContext#getStateStore() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AggregationProcessorSupplier.java    From rya with Apache License 2.0 6 votes vote down vote up
@Override
public void init(final ProcessorContext context) {
    this.context = context;

    // Sort the group by vars so that they will be written to the state store in the same order every time.
    final List<String> groupByVars = Lists.newArrayList(aggNode.getGroupBindingNames());
    groupByVars.sort(Comparator.naturalOrder());

    // Get a reference to the state store that keeps track of aggregation state.
    final KeyValueStore<String, AggregationState> stateStore =
            (KeyValueStore<String, AggregationState>) context.getStateStore( stateStoreName );
    aggStateStore = new KeyValueAggregationStateStore(stateStore, groupByVars);

    // Create the aggregation evaluator.
    evaluator = AggregationsEvaluator.make(aggStateStore, aggNode, groupByVars);
}
 
Example 2
Source File: MovingAverageProcessor.java    From kafka-streams-ex with MIT License 6 votes vote down vote up
/** Initializes the state store with the name "FAST-store", where
 * `type` is the type specified in the constructor.
 *
 * {@inheritDoc}
 */
@Override
@SuppressWarnings("unchecked")
public void init(ProcessorContext context) {
    
    this.context = context;

    // Schedules the `punctuate` method for every second.
    this.context.schedule(1000L);

    state = (KeyValueStore) context.getStateStore("FAST-store");

    // Print the contents of the state store.
    state.all().forEachRemaining( 
        kv -> System.out.println(System.currentTimeMillis() + 
                                 " INIT: " + kv.key + 
                                 ", " + kv.value));

}
 
Example 3
Source File: StockPerformanceMetricsTransformer.java    From kafka-streams-in-action with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void init(final ProcessorContext processorContext) {
    keyValueStore = (KeyValueStore) processorContext.getStateStore(stocksStateStore);
    this.processorContext = processorContext;
    
    this.processorContext.schedule(5000, PunctuationType.WALL_CLOCK_TIME, this::doPunctuate);


    final String tagKey = "task-id";
    final String tagValue = processorContext.taskId().toString();
    final String nodeName = "StockPerformanceProcessor_"+count.getAndIncrement();
   metricsSensor = processorContext.metrics().addLatencyAndThroughputSensor("transformer-node",
            nodeName, "stock-performance-calculation",
            Sensor.RecordingLevel.DEBUG,
            tagKey,
            tagValue);
}
 
Example 4
Source File: PregelComputation.java    From kafka-graphs with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void init(final ProcessorContext context) {
    this.context = context;
    this.localSolutionSetStore = (KeyValueStore<K, Tuple4<Integer, VV, Integer, VV>>) context.getStateStore(localSolutionSetStoreName);
    this.verticesStore = (TimestampedKeyValueStore<K, VV>) context.getStateStore(vertices.queryableStoreName());
    this.edgesStore = (TimestampedKeyValueStore<K, Map<K, EV>>) context.getStateStore(edgesGroupedBySource.queryableStoreName());
}
 
Example 5
Source File: JoinProcessorSupplier.java    From rya with Apache License 2.0 5 votes vote down vote up
@Override
public void init(final ProcessorContext context) {
    // Hold onto the context so that we can forward results.
    this.context = context;

    final String appId = context.applicationId();
    final UUID queryId = UuidUtils.extractUuidFromStringEnd(appId);

    // Get a reference to the state store that keeps track of what can be joined with.
    final KeyValueStore<String, VisibilityBindingSet> stateStore =
            (KeyValueStore<String, VisibilityBindingSet>) context.getStateStore( stateStoreName );
    joinStateStore = new KeyValueJoinStateStore( stateStore, queryId.toString(), joinVars, allVars );
}
 
Example 6
Source File: WordCountProcessor.java    From KafkaExample with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void init(ProcessorContext context) {
	this.context = context;
	this.context.schedule(1000);
	this.kvStore = (KeyValueStore<String, Integer>) context.getStateStore("Counts");
}
 
Example 7
Source File: CPUMetricStreamHandler.java    From kafka-streams-example with Apache License 2.0 5 votes vote down vote up
@Override
public void init(ProcessorContext pc) {

    this.pc = pc;
    this.pc.schedule(12000); //invoke punctuate every 12 seconds
    this.machineToAvgCPUUsageStore = (KeyValueStore<String, Double>) pc.getStateStore(AVG_STORE_NAME);
    this.machineToNumOfRecordsReadStore = (KeyValueStore<String, Integer>) pc.getStateStore(NUM_RECORDS_STORE_NAME);

    PROC_LOGGER.info("Processor initialized");
}
 
Example 8
Source File: MonitorProcessorSupplier.java    From DBus with Apache License 2.0 5 votes vote down vote up
@Override
public void init(ProcessorContext context) {
    super.init(context);
    // 两分钟执行一次punctuate方法
    context().schedule(2 * 60 * 1000);
    this.store = (KeyValueStore<String, String>) context.getStateStore("zkInfo");
}
 
Example 9
Source File: StockProcessor.java    From Kafka-Streams-Real-time-Stream-Processing with The Unlicense 5 votes vote down vote up
@Override
public void init(ProcessorContext processorContext) {
    this.stateStore = (KeyValueStore<String, TickerStack>)
        processorContext.getStateStore(STATE_STORE_NAME);

    processorContext.schedule(
        Duration.ofSeconds(AppConfigs.secondsDelay),
        PunctuationType.WALL_CLOCK_TIME,
        new StockPunctuator(processorContext, stateStore)
    );
}
 
Example 10
Source File: StockPerformanceTransformer.java    From kafka-streams-in-action with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void init(ProcessorContext processorContext) {
    keyValueStore = (KeyValueStore) processorContext.getStateStore(stateStoreName);
    StockPerformancePunctuator punctuator = new StockPerformancePunctuator(differentialThreshold, processorContext, keyValueStore);
    processorContext.schedule(15000, PunctuationType.STREAM_TIME, punctuator);
}
 
Example 11
Source File: ReferentialTransformer.java    From SkaETL with Apache License 2.0 5 votes vote down vote up
@Override
public void init(ProcessorContext context) {
    super.init(context);
    referentialStateStore = (KeyValueStore<String, Referential>) context.getStateStore(REFERENTIAL);

    context.schedule(5 * 60 * 1000, PunctuationType.WALL_CLOCK_TIME, (timestamp) -> flush());
}
 
Example 12
Source File: RewardsTransformer.java    From Kafka-Streams-Real-time-Stream-Processing with The Unlicense 4 votes vote down vote up
@Override
public void init(ProcessorContext processorContext) {
    this.stateStore = (KeyValueStore) processorContext.getStateStore(
            AppConfigs.REWARDS_STORE_NAME);
}
 
Example 13
Source File: StreamsTopologyProvider.java    From apicurio-registry with Apache License 2.0 4 votes vote down vote up
@Override
public void init(ProcessorContext context) {
    super.init(context);
    //noinspection unchecked
    store = (KeyValueStore<Long, Str.TupleValue>) context.getStateStore(storeName);
}
 
Example 14
Source File: StreamsUtils.java    From football-events with MIT License 4 votes vote down vote up
@Override
public void init(ProcessorContext context) {
    store = (KeyValueStore<String, D>)context.getStateStore(storeName);
}
 
Example 15
Source File: MetricDataTransformer.java    From adaptive-alerting with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void init(ProcessorContext context) {
    this.context = context;
    this.metricDataKeyValueStore = (KeyValueStore<String, MetricData>) context.getStateStore(stateStoreName);

    /*
     *  Requests to model-service to fetch detectors for metrics are throttled
     *  This is done by buffering them in kafka KV store and periodically clearing that buffer
     * */
    //TODO decide PUNCTUATION time
    this.context.schedule(200, PunctuationType.WALL_CLOCK_TIME, (timestamp) -> {

        if (metricDataKeyValueStore.approximateNumEntries() >= detectorMapper.optimalBatchSize()) {
            KeyValueIterator<String, MetricData> iter = this.metricDataKeyValueStore.all();
            Map<String, MetricData> cacheMissedMetrics = new HashMap<>();

            while (iter.hasNext()) {
                KeyValue<String, MetricData> entry = iter.next();
                cacheMissedMetrics.put(entry.key, entry.value);
                metricDataKeyValueStore.delete(entry.key);
            }
            iter.close();

            List<Map<String, String>> cacheMissedMetricTags = cacheMissedMetrics.values().stream().map(value -> value.getMetricDefinition().getTags().getKv()).collect(Collectors.toList());

            if (detectorMapper.isSuccessfulDetectorMappingLookup(cacheMissedMetricTags)) {

                cacheMissedMetrics.forEach((originalKey, metricData) -> {
                    List<Detector> detectors = detectorMapper.getDetectorsFromCache(metricData.getMetricDefinition());
                    if (!detectors.isEmpty()) {
                        context.forward(removeSalt(originalKey), new MapperResult(metricData, detectors));
                    }
                });
            }

        } else {
            log.trace("ES lookup skipped, as batch size is not optimum");
        }

        // commit the current processing progress
        context.commit();
    });

}
 
Example 16
Source File: KStreamsTopologyDescriptionParserTest.java    From netbeans-mmd-plugin with Apache License 2.0 4 votes vote down vote up
@Override
public void init(final ProcessorContext context) {
  store = (KeyValueStore) context.getStateStore(storeName);
}
 
Example 17
Source File: FindDistinctEvents.java    From kafka-tutorials with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public void init(final ProcessorContext context) {
    this.context = context;
    eventIdStore = (WindowStore<E, Long>) context.getStateStore(storeName);
}
 
Example 18
Source File: StreamsTopologyProvider.java    From apicurio-registry with Apache License 2.0 4 votes vote down vote up
@Override
public void init(ProcessorContext context) {
    this.context = context;
    //noinspection unchecked
    store = (KeyValueStore<String, Str.Data>) context.getStateStore(properties.getStorageStoreName());
}
 
Example 19
Source File: MovingAverageProcessor.java    From kafka-streams-ex with MIT License 3 votes vote down vote up
/** Initializes the state store with the name `type` + "_store", where
 * `type` is the type specified in the constructor.
 *
 * {@inheritDoc}
 */
@Override
@SuppressWarnings("unchecked")
public void init(ProcessorContext context) {
    
    this.context = context;

    // Schedules the `punctuate` method for every second.
    this.context.schedule(1000L);

    state = (KeyValueStore) context.getStateStore(type + "-store");

}