org.apache.kafka.streams.processor.TopologyBuilder Java Examples

public static void main(String[] args) throws IOException {
		Properties props = new Properties();
        props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount-processor");
        props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka0:19092");
        props.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG, "zookeeper0:12181/kafka");
        props.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass());
        props.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG, Serdes.Integer().getClass());
        props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
		
		TopologyBuilder builder = new TopologyBuilder();
		builder.addSource("SOURCE", new StringDeserializer(), new StringDeserializer(), "words")
				.addProcessor("WordCountProcessor", WordCountProcessor::new, "SOURCE")
				.addStateStore(Stores.create("Counts").withStringKeys().withIntegerValues().inMemory().build(), "WordCountProcessor")
//				.connectProcessorAndStateStores("WordCountProcessor", "Counts")
				.addSink("SINK", "count", new StringSerializer(), new IntegerSerializer(), "WordCountProcessor");
		
        KafkaStreams stream = new KafkaStreams(builder, props);
        stream.start();
        System.in.read();
        stream.close();
        stream.cleanUp();
	}
 

@Test
public void multiplePatterns_singleStatement() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Setup a topology.
    final String query = "SELECT * WHERE { "
            + "?person <urn:talksTo> ?otherPerson . "
            + "?person ?action <urn:Bob>"
            + "}";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create some statements where some generates SP results and others do not.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add( new VisibilityStatement(vf.createStatement(vf.createIRI("urn:Alice"), vf.createIRI("urn:talksTo"), vf.createIRI("urn:Bob")), "a") );

    // Show the correct binding set results from the job.
    final Set<VisibilityBindingSet> expected = new HashSet<>();

    final QueryBindingSet bs = new QueryBindingSet();
    bs.addBinding("person", vf.createIRI("urn:Alice"));
    bs.addBinding("action", vf.createIRI("urn:talksTo"));
    bs.addBinding("otherPerson", vf.createIRI("urn:Bob"));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Override
public KafkaStreams make(final String ryaInstance, final StreamsQuery query) throws KafkaStreamsFactoryException {
    requireNonNull(ryaInstance);
    requireNonNull(query);

    // Setup the Kafka Stream program.
    final Properties streamsProps = new Properties();

    // Configure the Kafka servers that will be talked to.
    streamsProps.setProperty(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServersConfig);

    // Use the Query ID as the Application ID to ensure we resume where we left off the last time this command was run.
    streamsProps.put(StreamsConfig.APPLICATION_ID_CONFIG, "RyaStreams-Query-" + query.getQueryId());

    // Always start at the beginning of the input topic.
    streamsProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

    // Setup the topology that processes the Query.
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, query.getQueryId());

    try {
        final TopologyBuilder topologyBuilder = topologyFactory.build(query.getSparql(), statementsTopic, resultsTopic, new RandomUUIDFactory());
        return new KafkaStreams(topologyBuilder, new StreamsConfig(streamsProps));
    } catch (final MalformedQueryException | TopologyBuilderException e) {
        throw new KafkaStreamsFactoryException("Could not create a KafkaStreams processing topology for query " + query.getQueryId(), e);
    }
}
 

@Test
public void showWithinWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Get the RDF model objects that will be used to build the query.
    final String sparql =
            "PREFIX time: <http://www.w3.org/2006/time/> \n"
                    + "PREFIX tempf: <" + TemporalIRIs.NAMESPACE + ">\n"
                    + "SELECT * \n"
                    + "WHERE { \n"
                    + "  <urn:time> time:atTime ?date .\n"
                    + " FILTER(tempf:within(?date, \"" + TIME.toString() + "/" + TIME_20.toString() + "\")) "
                    + "}";
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create the statements that will be input into the query.
    final List<VisibilityStatement> statements = getStatements();

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("date", VF.createLiteral(TIME_10.toString()));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Test
public void showAfterWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Get the RDF model objects that will be used to build the query.
    final String sparql =
            "PREFIX time: <http://www.w3.org/2006/time/> \n"
                    + "PREFIX tempf: <" + TemporalIRIs.NAMESPACE + ">\n"
                    + "SELECT * \n"
                    + "WHERE { \n"
                    + "  <urn:time> time:atTime ?date .\n"
                    + " FILTER(tempf:after(?date, \"" + TIME_10.toString() + "\")) "
                    + "}";
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create the statements that will be input into the query.
    final List<VisibilityStatement> statements = getStatements();

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("date", VF.createLiteral(TIME_20.toString()));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Test
public void showBeforeWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Get the RDF model objects that will be used to build the query.
    final String sparql =
            "PREFIX time: <http://www.w3.org/2006/time/> \n"
                    + "PREFIX tempf: <" + TemporalIRIs.NAMESPACE + ">\n"
                    + "SELECT * \n"
                    + "WHERE { \n"
                    + "  <urn:time> time:atTime ?date .\n"
                    + " FILTER(tempf:before(?date, \"" + TIME_10.toString() + "\")) "
                    + "}";
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create the statements that will be input into the query.
    final List<VisibilityStatement> statements = getStatements();

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("date", VF.createLiteral(TIME.toString()));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Test
public void showEqualsWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Get the RDF model objects that will be used to build the query.
    final String sparql =
            "PREFIX time: <http://www.w3.org/2006/time/> \n"
                    + "PREFIX tempf: <" + TemporalIRIs.NAMESPACE + ">\n"
                    + "SELECT * \n"
                    + "WHERE { \n"
                    + "  <urn:time> time:atTime ?date .\n"
                    + " FILTER(tempf:equals(?date, \"" + TIME.toString() + "\")) "
                    + "}";
    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create the statements that will be input into the query.
    final List<VisibilityStatement> statements = getStatements();

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("date", VF.createLiteral(TIME.toString()));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Test
public void showProcessorWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Get the RDF model objects that will be used to build the query.
    final String sparql =
            "SELECT * " +
            "WHERE { " +
                "FILTER(?age < 10)" +
                "?person <urn:age> ?age " +
            "}";

    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create the statements that will be input into the query.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add(new VisibilityStatement(vf.createStatement(vf.createIRI("urn:Bob"), vf.createIRI("urn:age"), vf.createLiteral(11)), "a"));
    statements.add(new VisibilityStatement(vf.createStatement(vf.createIRI("urn:Alice"), vf.createIRI("urn:age"), vf.createLiteral(9)), "a"));

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createIRI("urn:Alice"));
    bs.addBinding("age", vf.createLiteral(9));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

private TopologyBuilder processingTopologyBuilder() {

        StateStoreSupplier machineToAvgCPUUsageStore
                = Stores.create(AVG_STORE_NAME)
                        .withStringKeys()
                        .withDoubleValues()
                        .inMemory()
                        .build();

        StateStoreSupplier machineToNumOfRecordsReadStore
                = Stores.create(NUM_RECORDS_STORE_NAME)
                        .withStringKeys()
                        .withIntegerValues()
                        .inMemory()
                        .build();

        TopologyBuilder builder = new TopologyBuilder();

        builder.addSource(SOURCE_NAME, TOPIC_NAME)
                .addProcessor(PROCESSOR_NAME, new ProcessorSupplier() {
                    @Override
                    public Processor get() {
                        return new CPUCumulativeAverageProcessor();
                    }
                }, SOURCE_NAME)
                .addStateStore(machineToAvgCPUUsageStore, PROCESSOR_NAME)
                .addStateStore(machineToNumOfRecordsReadStore, PROCESSOR_NAME);

        LOGGER.info("Kafka streams processing topology ready");

        return builder;
    }
 

@Test
public void singlePattern_singleStatement() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Setup a topology.
    final String query = "SELECT * WHERE { ?person <urn:talksTo> ?otherPerson }";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create a statement that generate an SP result.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add( new VisibilityStatement(vf.createStatement(vf.createIRI("urn:Alice"), vf.createIRI("urn:talksTo"), vf.createIRI("urn:Bob")), "a") );

    // Show the correct binding set results from the job.
    final Set<VisibilityBindingSet> expected = new HashSet<>();

    final QueryBindingSet bs = new QueryBindingSet();
    bs.addBinding("person", vf.createIRI("urn:Alice"));
    bs.addBinding("otherPerson", vf.createIRI("urn:Bob"));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Test
public void manyJoins() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Setup a topology.
    final String query =
            "SELECT * WHERE { " +
                "?person <urn:talksTo> ?employee ." +
                "?employee <urn:worksAt> ?business ." +
                "?employee <urn:hourlyWage> ?wage ." +
            " }";
    final TopologyFactory factory = new TopologyFactory();
    final TopologyBuilder builder = factory.build(query, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create some statements that generate a bunch of right SP results.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add( new VisibilityStatement(
            vf.createStatement(vf.createIRI("urn:Alice"), vf.createIRI("urn:talksTo"), vf.createIRI("urn:Bob")), "a") );
    statements.add( new VisibilityStatement(
            vf.createStatement(vf.createIRI("urn:Bob"), vf.createIRI("urn:worksAt"), vf.createIRI("urn:BurgerJoint")), "a") );
    statements.add( new VisibilityStatement(
            vf.createStatement(vf.createIRI("urn:Bob"), vf.createIRI("urn:hourlyWage"), vf.createLiteral(7.25)), "a") );

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    bs.addBinding("person", vf.createIRI("urn:Alice"));
    bs.addBinding("employee", vf.createIRI("urn:Bob"));
    bs.addBinding("business", vf.createIRI("urn:BurgerJoint"));
    bs.addBinding("wage", vf.createLiteral(7.25));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

@Test
public void showProcessorWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Create a topology for the Query that will be tested.
    final String sparql =
            "SELECT (?person AS ?p) ?otherPerson " +
            "WHERE { " +
                "?person <urn:talksTo> ?otherPerson . " +
            "}";

    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Load some data into the input topic.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add( new VisibilityStatement(vf.createStatement(vf.createIRI("urn:Alice"), vf.createIRI("urn:talksTo"), vf.createIRI("urn:Bob")), "a") );

    // Show the correct binding set results from the job.
    final Set<VisibilityBindingSet> expected = new HashSet<>();

    final MapBindingSet expectedBs = new MapBindingSet();
    expectedBs.addBinding("p", vf.createIRI("urn:Alice"));
    expectedBs.addBinding("otherPerson", vf.createIRI("urn:Bob"));
    expected.add(new VisibilityBindingSet(expectedBs, "a"));

    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, Sets.newHashSet(expected), VisibilityBindingSetDeserializer.class);
}
 

@Test
public void showProcessorWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Get the RDF model objects that will be used to build the query.
    final String sparql =
            "PREFIX geo: <http://www.opengis.net/ont/geosparql#>\n"
                    + "PREFIX geof: <" + GEO + ">\n"
                    + "SELECT * \n"
                    + "WHERE { \n"
                    + "  <urn:event1> geo:asWKT ?point .\n"
                    + " FILTER(geof:sfWithin(?point, \"POLYGON((-3 -2, -3 2, 1 2, 1 -2, -3 -2))\"^^geo:wktLiteral)) "
                    + "}";

    // Setup a topology.
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, new RandomUUIDFactory());

    // Create the statements that will be input into the query.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = getStatements();

    // Make the expected results.
    final Set<VisibilityBindingSet> expected = new HashSet<>();
    final MapBindingSet bs = new MapBindingSet();
    final WKTWriter w = new WKTWriter();
    bs.addBinding("point", vf.createLiteral(w.write(ZERO), GeoConstants.XMLSCHEMA_OGC_WKT));
    expected.add( new VisibilityBindingSet(bs, "a") );

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityBindingSetDeserializer.class);
}
 

public static void main(String[] args) {

        StreamsConfig streamingConfig = new StreamsConfig(getProperties());

        TopologyBuilder builder = new TopologyBuilder();

        JsonSerializer<StockTransactionSummary> stockTxnSummarySerializer = new JsonSerializer<>();
        JsonDeserializer<StockTransactionSummary> stockTxnSummaryDeserializer = new JsonDeserializer<>(StockTransactionSummary.class);
        JsonDeserializer<StockTransaction> stockTxnDeserializer = new JsonDeserializer<>(StockTransaction.class);
        JsonSerializer<StockTransaction> stockTxnJsonSerializer = new JsonSerializer<>();
        StringSerializer stringSerializer = new StringSerializer();
        StringDeserializer stringDeserializer = new StringDeserializer();

        Serde<StockTransactionSummary> stockTransactionSummarySerde = Serdes.serdeFrom(stockTxnSummarySerializer,stockTxnSummaryDeserializer);

        builder.addSource("stocks-source", stringDeserializer, stockTxnDeserializer, "stocks")
                       .addProcessor("summary", StockSummaryProcessor::new, "stocks-source")
                       .addStateStore(Stores.create("stock-transactions").withStringKeys()
                               .withValues(stockTransactionSummarySerde).inMemory().maxEntries(100).build(),"summary")
                       .addSink("sink", "stocks-out", stringSerializer,stockTxnJsonSerializer,"stocks-source")
                       .addSink("sink-2", "transaction-summary", stringSerializer, stockTxnSummarySerializer, "summary");

        System.out.println("Starting StockSummaryStatefulProcessor Example");
        KafkaStreams streaming = new KafkaStreams(builder, streamingConfig);
        streaming.start();
        System.out.println("StockSummaryStatefulProcessor Example now started");

    }
 

public static void main(String[] args) throws Exception {

        StreamsConfig streamingConfig = new StreamsConfig(getProperties());

        JsonDeserializer<Purchase> purchaseJsonDeserializer = new JsonDeserializer<>(Purchase.class);
        JsonSerializer<Purchase> purchaseJsonSerializer = new JsonSerializer<>();
        JsonSerializer<RewardAccumulator> rewardAccumulatorJsonSerializer = new JsonSerializer<>();
        JsonSerializer<PurchasePattern> purchasePatternJsonSerializer = new JsonSerializer<>();

        StringDeserializer stringDeserializer = new StringDeserializer();
        StringSerializer stringSerializer = new StringSerializer();

        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.addSource("SOURCE", stringDeserializer, purchaseJsonDeserializer, "src-topic")

                .addProcessor("PROCESS", CreditCardAnonymizer::new, "SOURCE")
                .addProcessor("PROCESS2", PurchasePatterns::new, "PROCESS")
                .addProcessor("PROCESS3", CustomerRewards::new, "PROCESS")

                .addSink("SINK", "patterns", stringSerializer, purchasePatternJsonSerializer, "PROCESS2")
                .addSink("SINK2", "rewards",stringSerializer, rewardAccumulatorJsonSerializer, "PROCESS3")
                .addSink("SINK3", "purchases", stringSerializer, purchaseJsonSerializer, "PROCESS");

        System.out.println("Starting PurchaseProcessor Example");
        KafkaStreams streaming = new KafkaStreams(topologyBuilder, streamingConfig);
        streaming.start();
        System.out.println("Now started PurchaseProcessor Example");

    }
 

@Test
public void showProcessorWorks() throws Exception {
    // Enumerate some topics that will be re-used
    final String ryaInstance = UUID.randomUUID().toString();
    final UUID queryId = UUID.randomUUID();
    final String statementsTopic = KafkaTopics.statementsTopic(ryaInstance);
    final String resultsTopic = KafkaTopics.queryResultsTopic(ryaInstance, queryId);

    // Create a topology for the Query that will be tested.
    final String sparql =
            "CONSTRUCT {" +
                "_:b a <urn:movementObservation> ; " +
                "<urn:location> ?location ; " +
                "<urn:direction> ?direction ; " +
            "}" +
            "WHERE {" +
                "?thing <urn:corner> ?location ." +
                "?thing <urn:compass> ?direction." +
            "}";

    final String bNodeId = UUID.randomUUID().toString();
    final TopologyBuilder builder = new TopologyFactory().build(sparql, statementsTopic, resultsTopic, () -> bNodeId);

    // Create the statements that will be input into the query.
    final ValueFactory vf = SimpleValueFactory.getInstance();
    final List<VisibilityStatement> statements = new ArrayList<>();
    statements.add( new VisibilityStatement(
            vf.createStatement(vf.createIRI("urn:car1"), vf.createIRI("urn:compass"), vf.createIRI("urn:NW")), "a") );
    statements.add( new VisibilityStatement(
            vf.createStatement(vf.createIRI("urn:car1"), vf.createIRI("urn:corner"), vf.createIRI("urn:corner1")), "a") );

    // Make the expected results.
    final Set<VisibilityStatement> expected = new HashSet<>();
    final BNode blankNode = vf.createBNode(bNodeId);

    expected.add(new VisibilityStatement(vf.createStatement(blankNode, RDF.TYPE, vf.createIRI("urn:movementObservation")), "a"));
    expected.add(new VisibilityStatement(vf.createStatement(blankNode, vf.createIRI("urn:direction"), vf.createIRI("urn:NW")), "a"));
    expected.add(new VisibilityStatement(vf.createStatement(blankNode, vf.createIRI("urn:location"), vf.createIRI("urn:corner1")), "a"));

    // Run the test.
    RyaStreamsTestUtil.runStreamProcessingTest(kafka, statementsTopic, resultsTopic, builder, statements, expected, VisibilityStatementDeserializer.class);
}
 

/** Runs the streams program, writing to the "fast-avgs-instrumented", 
    * "medium-avgs-instrumented", and "slow-avgs-instrumented" topics.
    *
    * @param args Not used.
    */
public static void main(String[] args) throws Exception { 
       
       // Configuration for Kafka Streams.
       Properties config = new Properties();

       config.put(StreamsConfig.APPLICATION_ID_CONFIG,
                  "processor-kafka-streams-instrumented");
       config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG,
                  "localhost:9092");
       config.put(StreamsConfig.ZOOKEEPER_CONNECT_CONFIG,
                  "localhost:2181");
       config.put(StreamsConfig.KEY_SERDE_CLASS_CONFIG,
                  Serdes.String().getClass().getName());
       config.put(StreamsConfig.VALUE_SERDE_CLASS_CONFIG,
                  Serdes.Double().getClass().getName());

       // Start at latest message.
       config.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest");

       // Create the state stores. We need one for each of the
       // MessageProcessor's in the topology.
       StateStoreSupplier fastStore = 
           Stores.create("FAST-store")
                 .withStringKeys()
                 .withDoubleValues()
                 .inMemory()
                 .build();

       // Build the topology.
       TopologyBuilder builder = new TopologyBuilder();
       builder.addSource("messages-source",
                         Serdes.String().deserializer(),
                         Serdes.Double().deserializer(),
                         "messages-instrumented")
              .addProcessor("FAST-processor",
                            () -> new MovingAverageProcessor(0.1),
                            "messages-source")
              .addStateStore(fastStore, "FAST-processor")
              .addSink("FAST-sink", 
                       "fast-avgs-instrumented", 
                       Serdes.String().serializer(),
                       Serdes.Double().serializer(),
                       "FAST-processor");

       KafkaStreams streams = new KafkaStreams(builder, config);
       streams.start();

}
 

/**
 * Runs a Kafka Streams topology, loads statements into the input topic, read the binding sets that come out of
 * the results topic, and ensures the expected results match the read results.
 *
 * @param <T> The type of value that will be consumed from the results topic.
 * @param kafka - The embedded Kafka instance that is being tested with. (not null)
 * @param statementsTopic - The topic statements will be written to. (not null)
 * @param resultsTopic - The topic results will be read from. (not null)
 * @param builder - The streams topology that will be executed. (not null)
 * @param statements - The statements that will be loaded into the topic. (not null)
 * @param expected - The expected results. (not null)
 * @param expectedDeserializerClass - The class of the deserializer that will be used when reading
 *   values from the results topic. (not null)
 * @throws Exception If any exception was thrown while running the test.
 */
public static <T> void runStreamProcessingTest(
        final KafkaTestInstanceRule kafka,
        final String statementsTopic,
        final String resultsTopic,
        final TopologyBuilder builder,
        final List<VisibilityStatement> statements,
        final Set<T> expected,
        final Class<? extends Deserializer<T>> expectedDeserializerClass) throws Exception {
    requireNonNull(kafka);
    requireNonNull(statementsTopic);
    requireNonNull(resultsTopic);
    requireNonNull(builder);
    requireNonNull(statements);
    requireNonNull(expected);
    requireNonNull(expectedDeserializerClass);

    // Explicitly create the topics that are being used.
    kafka.createTopic(statementsTopic);
    kafka.createTopic(resultsTopic);

    // Start the streams program.
    final Properties props = kafka.createBootstrapServerConfig();
    props.put(StreamsConfig.APPLICATION_ID_CONFIG, UUID.randomUUID().toString());
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

    final KafkaStreams streams = new KafkaStreams(builder, new StreamsConfig(props));
    streams.cleanUp();
    try {
        streams.start();

        // Wait for the streams application to start. Streams only see data after their consumers are connected.
        Thread.sleep(6000);

        // Load the statements into the input topic.
        try(Producer<String, VisibilityStatement> producer = KafkaTestUtil.makeProducer(
                kafka, StringSerializer.class, VisibilityStatementSerializer.class)) {
            new KafkaLoadStatements(statementsTopic, producer).fromCollection(statements);
        }

        // Wait for the final results to appear in the output topic and verify the expected Binding Sets were found.
        try(Consumer<String, T> consumer = KafkaTestUtil.fromStartConsumer(kafka, StringDeserializer.class, expectedDeserializerClass)) {
            // Register the topic.
            consumer.subscribe(Arrays.asList(resultsTopic));

            // Poll for the result.
            final Set<T> results = Sets.newHashSet( KafkaTestUtil.pollForResults(500, 6, expected.size(), consumer) );

            // Show the correct binding sets results from the job.
            assertEquals(expected, results);
        }
    } finally {
        streams.close();
    }
}
 

@Override
public TopologyBuilder build(
        final String sparqlQuery,
        final String statementsTopic,
        final String resultsTopic,
        final BNodeIdFactory bNodeIdFactory)
        throws MalformedQueryException, TopologyBuilderException {
    requireNonNull(sparqlQuery);
    requireNonNull(statementsTopic);
    requireNonNull(resultsTopic);

    final ParsedQuery parsedQuery = new SPARQLParser().parseQuery(sparqlQuery, null);
    final TopologyBuilder builder = new TopologyBuilder();

    final TupleExpr expr = parsedQuery.getTupleExpr();
    final QueryVisitor visitor = new QueryVisitor(bNodeIdFactory);
    expr.visit(visitor);

    processorEntryList = visitor.getProcessorEntryList();
    final Map<TupleExpr, String> idMap = visitor.getIDs();
    // add source node
    builder.addSource(SOURCE, new StringDeserializer(), new VisibilityStatementDeserializer(), statementsTopic);

    // processing the processor entry list in reverse order means we go from leaf
    // nodes -> parent nodes.
    // So, when the parent processing nodes get added, the upstream
    // processing node will already exist.

    ProcessorEntry entry = null;
    for (int ii = processorEntryList.size() - 1; ii >= 0; ii--) {
        entry = processorEntryList.get(ii);
        //statement patterns need to be connected to the Source.
        if(entry.getNode() instanceof StatementPattern) {
            builder.addProcessor(entry.getID(), entry.getSupplier(), SOURCE);
        } else {
            final List<TupleExpr> parents = entry.getUpstreamNodes();
            final String[] parentIDs = new String[parents.size()];
            for (int id = 0; id < parents.size(); id++) {
                parentIDs[id] = idMap.get(parents.get(id));
            }
            builder.addProcessor(entry.getID(), entry.getSupplier(), parentIDs);
        }

        // Add a state store for any node type that requires one.
        if (entry.getNode() instanceof Join ||  entry.getNode() instanceof LeftJoin || entry.getNode() instanceof Group) {
            // Add a state store for the join processor.
            final StateStoreSupplier joinStoreSupplier =
                    Stores.create( entry.getID() )
                        .withStringKeys()
                        .withValues(new VisibilityBindingSetSerde())
                        .persistent()
                        .build();
            builder.addStateStore(joinStoreSupplier, entry.getID());
        }
    }

    if (entry == null) {
        throw new TopologyBuilderException("No valid processor entries found.");
    }

    // Add a formatter that converts the ProcessorResults into the output format.
    final SinkEntry<?,?> sinkEntry = visitor.getSinkEntry();
    builder.addProcessor("OUTPUT_FORMATTER", sinkEntry.getFormatterSupplier(), entry.getID());

    // Add the sink.
    builder.addSink(SINK, resultsTopic, sinkEntry.getKeySerializer(), sinkEntry.getValueSerializer(), "OUTPUT_FORMATTER");

    return builder;
}
 

/**
 * Builds a {@link TopologyBuilder} based on the provided SPARQL query that
 * pulls from {@code statementsTopic} for input and writes the query's results
 * to {@code resultsTopic}.
 *
 * @param sparqlQuery - The SPARQL query to build a topology for. (not null)
 * @param statementsTopic - The topic for the source to read from. (not null)
 * @param resultsTopic - The topic for the sink to write to. (not null)
 * @param bNodeIdFactory - A factory that generates Blank Node IDs if any are required. (not null)
 * @return The created {@link TopologyBuilder}.
 * @throws MalformedQueryException - The provided query is not a valid SPARQL query.
 * @throws TopologyBuilderException - A problem occurred while constructing the topology.
 */
public TopologyBuilder build(
        final String sparqlQuery,
        final String statementsTopic,
        final String resultsTopic,
        final BNodeIdFactory bNodeIdFactory) throws MalformedQueryException, TopologyBuilderException;