Java Code Examples for io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde#configure()

The following examples show how to use io.confluent.kafka.streams.serdes.avro.SpecificAvroSerde#configure() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AggregatingCountTest.java From kafka-tutorials with Apache License 2.0

7 votes

private SpecificAvroSerde<TicketSale> makeSerializer(Properties envProps)
    throws IOException, RestClientException {

  final MockSchemaRegistryClient client = new MockSchemaRegistryClient();
  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final Schema schema = TicketSale.SCHEMA$;
  client.register(inputTopic + "-value", schema);
  client.register(outputTopic + "-value", schema);

  SpecificAvroSerde<TicketSale> serde = new SpecificAvroSerde<>(client);

  Map<String, String> config = new HashMap<>();
  config.put("schema.registry.url", envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);

  return serde;
}

Example 2

Source File: AggregatingSumTest.java From kafka-tutorials with Apache License 2.0

7 votes

private SpecificAvroSerde<TicketSale> makeSerializer(Properties envProps)
    throws IOException, RestClientException {

  final MockSchemaRegistryClient client = new MockSchemaRegistryClient();
  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final Schema schema = TicketSale.SCHEMA$;
  client.register(inputTopic + "-value", schema);
  client.register(outputTopic + "-value", schema);

  SpecificAvroSerde<TicketSale> serde = new SpecificAvroSerde<>(client);

  Map<String, String> config = new HashMap<>();
  config.put("schema.registry.url", envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);

  return serde;
}

Example 3

Source File: StreamsIngestTest.java From kafka-tutorials with Apache License 2.0

6 votes

private SpecificAvroSerde<City> makeSerializer(Properties envProps)
    throws IOException, RestClientException {

  final MockSchemaRegistryClient client = new MockSchemaRegistryClient();
  String inputTopic = envProps.getProperty("input.topic.name");
  String outputTopic = envProps.getProperty("output.topic.name");

  final Schema schema = City.SCHEMA$;
  client.register(inputTopic + "-value", schema);
  client.register(outputTopic + "-value", schema);

  SpecificAvroSerde<City> serde = new SpecificAvroSerde<>(client);

  Map<String, String> config = new HashMap<>();
  config.put("schema.registry.url", envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);

  return serde;
}

Example 4

Source File: FkJoinTableToTable.java From kafka-tutorials with Apache License 2.0

5 votes

static <T extends SpecificRecord> SpecificAvroSerde<T> getSpecificAvroSerde(final Properties envProps) {
    final SpecificAvroSerde<T> specificAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}

Example 5

Source File: DynamicOutputTopic.java From kafka-tutorials with Apache License 2.0

5 votes

static <T extends SpecificRecord> SpecificAvroSerde<T> getSpecificAvroSerde(final Properties envProps) {
    final SpecificAvroSerde<T> specificAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}

Example 6

Source File: CogroupingStreams.java From kafka-tutorials with Apache License 2.0

5 votes

static <T extends SpecificRecord> SpecificAvroSerde<T> getSpecificAvroSerde(final Properties envProps) {
    final SpecificAvroSerde<T> specificAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaSchemaSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    specificAvroSerde.configure(serdeConfig, false);
    return specificAvroSerde;
}

Example 7

Source File: StreamsIngest.java From kafka-tutorials with Apache License 2.0

5 votes

private SpecificAvroSerde<City> citySerde(final Properties envProps) {
  final SpecificAvroSerde<City> serde = new SpecificAvroSerde<>();
  Map<String, String> config = new HashMap<>();
  config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);
  return serde;
}

Example 8

Source File: FindDistinctEvents.java From kafka-tutorials with Apache License 2.0

5 votes

private SpecificAvroSerde<Click> buildClicksSerde(final Properties envProps) {
    final SpecificAvroSerde<Click> serde = new SpecificAvroSerde<>();
    Map<String, String> config = new HashMap<>();
    config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
    serde.configure(config, false);
    return serde;
}

Example 9

Source File: AggregatingSum.java From kafka-tutorials with Apache License 2.0

5 votes

private SpecificAvroSerde<TicketSale> ticketSaleSerde(final Properties envProps) {
  final SpecificAvroSerde<TicketSale> serde = new SpecificAvroSerde<>();
  Map<String, String> config = new HashMap<>();
  config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);
  return serde;
}

Example 10

Source File: TumblingWindow.java From kafka-tutorials with Apache License 2.0

5 votes

private SpecificAvroSerde<Rating> ratedMovieAvroSerde(Properties envProps) {
    SpecificAvroSerde<Rating> movieAvroSerde = new SpecificAvroSerde<>();

    final HashMap<String, String> serdeConfig = new HashMap<>();
    serdeConfig.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
            envProps.getProperty("schema.registry.url"));

    movieAvroSerde.configure(serdeConfig, false);
    return movieAvroSerde;
}

Example 11

Source File: CountVersionApplication.java From spring-cloud-stream-samples with Apache License 2.0

5 votes

@Bean
public Function<KStream<Object, Sensor>, KStream<String, Long>> process() {

	//The following Serde definitions are not needed in the topoloyy below
	//as we are not using it. However, if your topoloyg explicitly uses this
	//Serde, you need to configure this with the schema registry url as below.

	final Map<String, String> serdeConfig = Collections.singletonMap(
			AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");

	final SpecificAvroSerde<Sensor> sensorSerde = new SpecificAvroSerde<>();
	sensorSerde.configure(serdeConfig, false);

	return input -> input
			.map((k, value) -> {
				String newKey = "v1";
				if (value.getId().toString().endsWith("v2")) {
					newKey = "v2";
				}
				return new KeyValue<>(newKey, value);
			})
			.groupByKey()
			.count(Materialized.<String, Long, KeyValueStore<Bytes, byte[]>>as(STORE_NAME)
					.withKeySerde(Serdes.String())
					.withValueSerde(Serdes.Long()))
			.toStream();

}

Example 12

Source File: AggregatingMinMaxTest.java From kafka-tutorials with Apache License 2.0

5 votes

private static SpecificAvroSerde<YearlyMovieFigures> makeYearlyMovieFiguresSerializer(
        Properties envProps, SchemaRegistryClient srClient) {

  SpecificAvroSerde<YearlyMovieFigures> serde = new SpecificAvroSerde<>(srClient);
  serde.configure(Collections.singletonMap(
          "schema.registry.url", envProps.getProperty("schema.registry.url")), false);
  return serde;
}

Example 13

Source File: AggregatingMinMaxTest.java From kafka-tutorials with Apache License 2.0

5 votes

private static SpecificAvroSerde<MovieTicketSales> makeMovieTicketSalesSerializer(
        Properties envProps, SchemaRegistryClient srClient) {

  SpecificAvroSerde<MovieTicketSales> serde = new SpecificAvroSerde<>(srClient);
  serde.configure(Collections.singletonMap(
          "schema.registry.url", envProps.getProperty("schema.registry.url")), false);
  return serde;
}

Example 14

Source File: AggregatingMinMax.java From kafka-tutorials with Apache License 2.0

5 votes

public static SpecificAvroSerde<YearlyMovieFigures> movieFiguresSerde(final Properties envProps) {
  final SpecificAvroSerde<YearlyMovieFigures> serde = new SpecificAvroSerde<>();
  serde.configure(Collections.singletonMap(
    AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url")
  ), false);
  return serde;
}

Example 15

Source File: AggregatingMinMax.java From kafka-tutorials with Apache License 2.0

5 votes

public static SpecificAvroSerde<MovieTicketSales> ticketSaleSerde(final Properties envProps) {
  final SpecificAvroSerde<MovieTicketSales> serde = new SpecificAvroSerde<>();
  serde.configure(Collections.singletonMap(
          AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG,
          envProps.getProperty("schema.registry.url")), false);
  return serde;
}

Example 16

Source File: FilterEvents.java From kafka-tutorials with Apache License 2.0

5 votes

private SpecificAvroSerde<Publication> publicationSerde(final Properties envProps) {
  final SpecificAvroSerde<Publication> serde = new SpecificAvroSerde<>();
  Map<String, String> config = new HashMap<>();
  config.put(AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, envProps.getProperty("schema.registry.url"));
  serde.configure(config, false);
  return serde;
}

Example 17

Source File: RunningAverage.java From kafka-tutorials with Apache License 2.0

4 votes

public static SpecificAvroSerde<CountAndSum> getCountAndSumSerde(Properties envProps) {
  SpecificAvroSerde<CountAndSum> serde = new SpecificAvroSerde<>();
  serde.configure(getSerdeConfig(envProps), false);
  return serde;
}

Example 18

Source File: RunningAverage.java From kafka-tutorials with Apache License 2.0

4 votes

public static SpecificAvroSerde<Rating> getRatingSerde(Properties envProps) {
  SpecificAvroSerde<Rating> serde = new SpecificAvroSerde<>();
  serde.configure(getSerdeConfig(envProps), false);
  return serde;
}

Example 19

Source File: KafkaStreamsInteractiveQuerySample.java From spring-cloud-stream-samples with Apache License 2.0

4 votes

@Bean
public BiConsumer<KStream<String, PlayEvent>, KTable<Long, Song>> process() {

	return (s, t) -> {
		// create and configure the SpecificAvroSerdes required in this example
		final Map<String, String> serdeConfig = Collections.singletonMap(
				AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG, "http://localhost:8081");

		final SpecificAvroSerde<PlayEvent> playEventSerde = new SpecificAvroSerde<>();
		playEventSerde.configure(serdeConfig, false);

		final SpecificAvroSerde<Song> keySongSerde = new SpecificAvroSerde<>();
		keySongSerde.configure(serdeConfig, true);

		final SpecificAvroSerde<Song> valueSongSerde = new SpecificAvroSerde<>();
		valueSongSerde.configure(serdeConfig, false);

		final SpecificAvroSerde<SongPlayCount> songPlayCountSerde = new SpecificAvroSerde<>();
		songPlayCountSerde.configure(serdeConfig, false);

		// Accept play events that have a duration >= the minimum
		final KStream<Long, PlayEvent> playsBySongId =
				s.filter((region, event) -> event.getDuration() >= MIN_CHARTABLE_DURATION)
						// repartition based on song id
						.map((key, value) -> KeyValue.pair(value.getSongId(), value));

		// join the plays with song as we will use it later for charting
		final KStream<Long, Song> songPlays = playsBySongId.leftJoin(t,
				(value1, song) -> song,
				Joined.with(Serdes.Long(), playEventSerde, valueSongSerde));

		// create a state store to track song play counts
		final KTable<Song, Long> songPlayCounts = songPlays.groupBy((songId, song) -> song,
				Serialized.with(keySongSerde, valueSongSerde))
				.count(Materialized.<Song, Long, KeyValueStore<Bytes, byte[]>>as(SONG_PLAY_COUNT_STORE)
						.withKeySerde(valueSongSerde)
						.withValueSerde(Serdes.Long()));

		final TopFiveSerde topFiveSerde = new TopFiveSerde();

		// Compute the top five charts for each genre. The results of this computation will continuously update the state
		// store "top-five-songs-by-genre", and this state store can then be queried interactively via a REST API (cf.
		// MusicPlaysRestService) for the latest charts per genre.
		songPlayCounts.groupBy((song, plays) ->
						KeyValue.pair(song.getGenre().toLowerCase(),
								new SongPlayCount(song.getId(), plays)),
				Serialized.with(Serdes.String(), songPlayCountSerde))
				// aggregate into a TopFiveSongs instance that will keep track
				// of the current top five for each genre. The data will be available in the
				// top-five-songs-genre store
				.aggregate(TopFiveSongs::new,
						(aggKey, value, aggregate) -> {
							aggregate.add(value);
							return aggregate;
						},
						(aggKey, value, aggregate) -> {
							aggregate.remove(value);
							return aggregate;
						},
						Materialized.<String, TopFiveSongs, KeyValueStore<Bytes, byte[]>>as(TOP_FIVE_SONGS_BY_GENRE_STORE)
								.withKeySerde(Serdes.String())
								.withValueSerde(topFiveSerde)
				);

		// Compute the top five chart. The results of this computation will continuously update the state
		// store "top-five-songs", and this state store can then be queried interactively via a REST API (cf.
		// MusicPlaysRestService) for the latest charts per genre.
		songPlayCounts.groupBy((song, plays) ->
						KeyValue.pair(TOP_FIVE_KEY,
								new SongPlayCount(song.getId(), plays)),
				Serialized.with(Serdes.String(), songPlayCountSerde))
				.aggregate(TopFiveSongs::new,
						(aggKey, value, aggregate) -> {
							aggregate.add(value);
							return aggregate;
						},
						(aggKey, value, aggregate) -> {
							aggregate.remove(value);
							return aggregate;
						},
						Materialized.<String, TopFiveSongs, KeyValueStore<Bytes, byte[]>>as(TOP_FIVE_SONGS_STORE)
								.withKeySerde(Serdes.String())
								.withValueSerde(topFiveSerde)
				);
	};

}

Example 20

Source File: WindowFinalResult.java From kafka-tutorials with Apache License 2.0

3 votes

public static void main(String[] args) {

        final Config config = ConfigFactory.load();

        final Properties properties = buildProperties(config);

        Map<String, Object> serdeConfig =
                singletonMap(SCHEMA_REGISTRY_URL_CONFIG, config.getString("schema.registry.url"));

        SpecificAvroSerde<PressureAlert> pressureSerde = new SpecificAvroSerde<>();

        pressureSerde.configure(serdeConfig, false);

        TimeWindows windows = TimeWindows

                .of(config.getDuration("window.size"))

                .advanceBy(config.getDuration("window.size"))

                .grace(config.getDuration("window.grace.period"));

        Topology topology = buildTopology(config, windows, pressureSerde);

        logger.debug(topology.describe().toString());

        final KafkaStreams streams = new KafkaStreams(topology, properties);

        Runtime.getRuntime().addShutdownHook(new Thread(streams::close));

        streams.cleanUp();
        streams.start();
    }