org.apache.kafka.common.serialization.StringDeserializer Java Examples

The following examples show how to use org.apache.kafka.common.serialization.StringDeserializer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConsumerPropertiesBuilderTest.java    From remoting-kafka-plugin with MIT License 7 votes vote down vote up 
@Test
public void testBuildNoAutoOffsetReset() throws RemotingKafkaConfigurationException {
    Properties props = new ConsumerPropertiesBuilder()
            .withBootstrapServers("localhost:9092")
            .withEnableAutoCommit(false)
            .withGroupID("test")
            .withKeyDeserializer(StringDeserializer.class)
            .withValueDeserializer(StringDeserializer.class)
            .build();
    assertEquals("localhost:9092", props.get(KafkaConfigs.BOOTSTRAP_SERVERS));
    assertEquals(false, props.get(KafkaConfigs.ENABLE_AUTO_COMMIT));
    assertEquals(null, props.get(KafkaConfigs.AUTO_OFFSET_RESET));
    assertEquals("test", props.get(KafkaConfigs.GROUP_ID));
    assertEquals(StringDeserializer.class, props.get(KafkaConfigs.KEY_DESERIALIZER));
    assertEquals(StringDeserializer.class, props.get(KafkaConfigs.VALUE_DESERIALIZER));
}
Example #2
Source File: KafkaConsumerConfig.java    From ext-opensource-netty with Mozilla Public License 2.0 6 votes vote down vote up 
private Map<String, Object> consumerConfigs() {
    Map<String, Object> propsMap = new HashMap<>(16);
    propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
    String sId = groupId;
    if (groupId == null || (groupId.trim().length() <= 0)) {
    	sId = StringsUtil.getUuid();
    }
    propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, sId);
    propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,enableAutoCommit);
    propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,autoCommitInterval);
    
    ///latest,none,earliest
    propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
    propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    return propsMap;
}
Example #3
Source File: TrafficControlIntegrationTest.java    From data-highway with Apache License 2.0 6 votes vote down vote up 
private KafkaConsumer<String, String> createKafkaConsumer(
    String bootstrapServers,
    String patchTopic,
    String patchGroupId,
    ObjectMapper objectMapper) {
  Properties properties = new Properties();
  properties.setProperty("bootstrap.servers", bootstrapServers);
  properties.setProperty("group.id", patchGroupId);
  properties.setProperty("auto.offset.reset", "earliest");
  properties.setProperty("enable.auto.commit", "false");
  KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<>(
      properties,
      new StringDeserializer(),
      new StringDeserializer());
  kafkaConsumer.subscribe(Lists.newArrayList(patchTopic));
  return kafkaConsumer;
}
Example #4
Source File: StreamsSelectAndProjectIntTest.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up 
private void testSelectWithFilter(String resultStream,
                                  String inputStreamName,
                                  DataSource.DataSourceSerDe dataSourceSerDe) throws Exception {

  ksqlContext.sql(String.format("CREATE STREAM %s AS SELECT * FROM %s WHERE ORDERUNITS > 40;",
                                resultStream, inputStreamName));

  Map<String, GenericRow> results = testHarness.consumeData(resultStream,
                                                            dataProvider.schema(),
                                                            4,
                                                            new StringDeserializer(),
                                                            IntegrationTestHarness
                                                                .RESULTS_POLL_MAX_TIME_MS,
                                                            dataSourceSerDe);

  Assert.assertEquals(4, results.size());
}
Example #5
Source File: KafkaStreamsYellingIntegrationTest.java    From kafka-streams-in-action with Apache License 2.0 6 votes vote down vote up 
@Before
public void setUp() {
    Properties properties = StreamsTestUtils.getStreamsConfig("integrationTest",
            EMBEDDED_KAFKA.bootstrapServers(),
            STRING_SERDE_CLASSNAME,
            STRING_SERDE_CLASSNAME,
            new Properties());
    properties.put(IntegrationTestUtils.INTERNAL_LEAVE_GROUP_ON_CLOSE, true);
    
    streamsConfig = new StreamsConfig(properties);

    producerConfig = TestUtils.producerConfig(EMBEDDED_KAFKA.bootstrapServers(),
            StringSerializer.class,
            StringSerializer.class);

    consumerConfig = TestUtils.consumerConfig(EMBEDDED_KAFKA.bootstrapServers(),
            StringDeserializer.class,
            StringDeserializer.class);
}
Example #6
Source File: SampleRawConsumer.java    From kafka-encryption with Apache License 2.0 6 votes vote down vote up 
@Override
public void run() {

    Properties consumerProperties = new Properties();
    consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "sampleraw");
    consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

    try (KafkaConsumer<Long, String> consumer = new KafkaConsumer<Long, String>(
            consumerProperties,
            new LongDeserializer(),
            new StringDeserializer())) {

        consumer.subscribe(Collections.singleton("sampletopic"));
        for (; true; ) {
            ConsumerRecords<Long, String> records = consumer.poll(1000L);
            records.forEach(
                    record -> System.out.println(
                    "-------------------------------------------------------------\n" +
                    "raw record: key=" + record.key() + ", offset=" + record.offset() + ", value=" + record.value() +
                    "\n-------------------------------------------------------------\n\n")
            );
        }
    }
}
Example #7
Source File: ReadKafkaIntoHazelcast.java    From hazelcast-jet-demos with Apache License 2.0 6 votes vote down vote up 
public static Pipeline build(String bootstrapServers) {
    Properties properties = new Properties();
    properties.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
    properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
    properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());
    properties.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

    Pipeline pipeline = Pipeline.create();

    pipeline
            .readFrom(KafkaSources.kafka(properties, Constants.TOPIC_NAME_PRECIOUS))
            .withoutTimestamps()
            .writeTo(Sinks.map(Constants.IMAP_NAME_PRECIOUS));

    return pipeline;
}
Example #8
Source File: ConsumerPropertiesBuilderTest.java    From remoting-kafka-plugin with MIT License 6 votes vote down vote up 
@Test
public void testBuildAllConfigs() throws RemotingKafkaConfigurationException {
    Properties props = new ConsumerPropertiesBuilder()
            .withBootstrapServers("localhost:9092")
            .withEnableAutoCommit(false)
            .withAutoOffsetReset(AutoOffsetReset.EARLIEST)
            .withGroupID("test")
            .withKeyDeserializer(StringDeserializer.class)
            .withValueDeserializer(StringDeserializer.class)
            .build();
    assertEquals("localhost:9092", props.get(KafkaConfigs.BOOTSTRAP_SERVERS));
    assertEquals(false, props.get(KafkaConfigs.ENABLE_AUTO_COMMIT));
    assertEquals(AutoOffsetReset.EARLIEST.toString(), props.get(KafkaConfigs.AUTO_OFFSET_RESET));
    assertEquals("test", props.get(KafkaConfigs.GROUP_ID));
    assertEquals(StringDeserializer.class, props.get(KafkaConfigs.KEY_DESERIALIZER));
    assertEquals(StringDeserializer.class, props.get(KafkaConfigs.VALUE_DESERIALIZER));
}
Example #9
Source File: SampleRawConsumer.java    From kafka-encryption with Apache License 2.0 6 votes vote down vote up 
@Override
public void run() {

    Properties consumerProperties = new Properties();
    consumerProperties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    consumerProperties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "sampleraw");
    consumerProperties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");

    try (KafkaConsumer<Long, String> consumer = new KafkaConsumer<Long, String>(
            consumerProperties,
            new LongDeserializer(),
            new StringDeserializer())) {

        consumer.subscribe(Collections.singleton("sampletopic"));
        for (; true; ) {
            ConsumerRecords<Long, String> records = consumer.poll(1000L);
            records.forEach(
                    record -> System.out.println(
                        "-------------------------------------------------------------\n" +
                        "raw record: key=" + record.key() + ", offset=" + record.offset() + ", value=" + record.value() +
                        "\n-------------------------------------------------------------\n\n"
                    )
            );
        }
    }
}
Example #10
Source File: ConsumerTTL.java    From kafka_book_demo with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) {
    Properties props = new Properties();
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG,
            StringDeserializer.class.getName());
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,
            StringDeserializer.class.getName());
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
    props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    props.put(ConsumerConfig.INTERCEPTOR_CLASSES_CONFIG,
            ConsumerInterceptorTTL.class.getName());

    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Collections.singletonList(topic));

    while (true) {
        ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
        for (ConsumerRecord<String, String> record : records) {
            System.out.println(record.partition() + ":" + record.offset() + ":" + record.value());
        }
    }
}
Example #11
Source File: SimpleKafkaConsumer.java    From joyqueue with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:50088");
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "test_app");
    props.put(ConsumerConfig.CLIENT_ID_CONFIG, "test_app");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

    KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Arrays.asList("test_topic_0"));

    while (true) {
        ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000 * 1));
        for (ConsumerRecord<String, String> record : records) {
            System.out.println(String.format("record, key: %s, value: %s, offset: %s", record.key(), record.value(), record.offset()));
        }
    }
}
Example #12
Source File: SimpleKafkaConsumer.java    From joyqueue with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) {
        Properties props = new Properties();
        props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, KafkaConfigs.BOOTSTRAP);
        props.put(ConsumerConfig.GROUP_ID_CONFIG, KafkaConfigs.GROUP_ID);
        props.put(ConsumerConfig.CLIENT_ID_CONFIG, KafkaConfigs.GROUP_ID);
        props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(props);
        consumer.subscribe(Arrays.asList("test_topic_0"));

        while (true) {
//            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000 * 1));
            ConsumerRecords<String, String> records = consumer.poll(1000 * 1);
            for (ConsumerRecord<String, String> record : records) {
//                System.out.println(String.format("record, key: %s, value: %s, offset: %s, timestamp: %s", record.key(), record.value(), record.offset(), record.timestamp()));
                System.out.println(String.format("record, key: %s, value: %s, offset: %s", record.key(), record.value(), record.offset()));
            }
        }
    }
Example #13
Source File: KafkaConnector.java    From fasten with Apache License 2.0 6 votes vote down vote up 
/**
 * Returns Kafka properties.
 *
 * @param serverAddresses broker address
 * @param groupId         group id
 * @return Kafka Properties
 */
public static Properties kafkaConsumerProperties(List<String> serverAddresses, String groupId) {
    String deserializer = StringDeserializer.class.getName();
    Properties properties = new Properties();

    properties.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
            String.join(",", serverAddresses));
    properties.setProperty(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    properties.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, groupId + "_client");
    properties.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, deserializer);
    properties.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, deserializer);
    properties.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    properties.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
    properties.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "5");

    // Gives more time to the consumer for processing the records so
    // that the broker will NOT kill the consumer.
    properties.setProperty(ConsumerConfig.HEARTBEAT_INTERVAL_MS_CONFIG, "200000");
    properties.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "700000");

    return properties;
}
Example #14
Source File: KeycloakClientCredentialsWithJwtValidationAuthzTest.java    From strimzi-kafka-oauth with Apache License 2.0 6 votes vote down vote up 
static Properties buildConsumerConfig(String accessToken) {
    Properties p = new Properties();
    p.setProperty("security.protocol", "SASL_PLAINTEXT");
    p.setProperty("sasl.mechanism", "OAUTHBEARER");
    p.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required " +
            " oauth.access.token=\"" + accessToken + "\";");
    p.setProperty("sasl.login.callback.handler.class", "io.strimzi.kafka.oauth.client.JaasClientOauthLoginCallbackHandler");

    p.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka:9092");
    p.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    p.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

    p.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "consumer-group");
    p.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "10");
    p.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");

    return p;
}
Example #15
Source File: HydraClientCredentialsWithJwtValidationTest.java    From strimzi-kafka-oauth with Apache License 2.0 6 votes vote down vote up 
private static Properties buildConsumerConfig() {
    Properties p = new Properties();
    p.setProperty("security.protocol", "SASL_PLAINTEXT");
    p.setProperty("sasl.mechanism", "OAUTHBEARER");
    p.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required ;");
    p.setProperty("sasl.login.callback.handler.class", "io.strimzi.kafka.oauth.client.JaasClientOauthLoginCallbackHandler");

    p.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "kafka:9092");
    p.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    p.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

    p.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "consumer-group");
    p.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "10");
    p.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");

    return p;
}
Example #16
Source File: ExampleConsumer.java    From strimzi-kafka-oauth with Apache License 2.0 6 votes vote down vote up 
/**
 * Build KafkaConsumer properties. The specified values are defaults that can be overridden
 * through runtime system properties or env variables.
 *
 * @return Configuration properties
 */
private static Properties buildConsumerConfig() {

    Properties p = new Properties();

    p.setProperty("security.protocol", "SASL_PLAINTEXT");
    p.setProperty("sasl.mechanism", "OAUTHBEARER");
    p.setProperty("sasl.jaas.config", "org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required ;");
    p.setProperty("sasl.login.callback.handler.class", "io.strimzi.kafka.oauth.client.JaasClientOauthLoginCallbackHandler");

    p.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092");
    p.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    p.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());

    p.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "a_consumer-group");
    p.setProperty(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "10");
    p.setProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");

    return ConfigProperties.resolve(p);
}
Example #17
Source File: ProtostuffDeserializer.java    From BigData-In-Practice with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) {
    String brokerList = "192.168.0.101:9092";
    String topic = "topic.serialization";
    String groupId = "group.demo";
    Properties properties = new Properties();
    properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ProtostuffDeserializer.class.getName());
    properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
    properties.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);

    KafkaConsumer<String, Company> consumer = new KafkaConsumer<String, Company>(properties);
    consumer.subscribe(Collections.singletonList(topic));

    while (true) {
        ConsumerRecords<String, Company> records = consumer.poll(Duration.ofMillis(1000));
        for (ConsumerRecord record : records) {
            System.out.println(String.format("%s-%s-%s-%s",
                    record.topic(), record.partition(), record.offset(), record.value()));
            // 成功反序列化，输出：topic.serialization-0-1-Company(name=whirly, address=中国)
        }
    }
}
Example #18
Source File: ConsumerPropertiesBuilderTest.java    From remoting-kafka-plugin with MIT License 6 votes vote down vote up 
@Test
public void testBuildNoEnableAutoCommit() throws RemotingKafkaConfigurationException {
    Properties props = new ConsumerPropertiesBuilder()
            .withBootstrapServers("localhost:9092")
            .withAutoOffsetReset(AutoOffsetReset.EARLIEST)
            .withGroupID("test")
            .withKeyDeserializer(StringDeserializer.class)
            .withValueDeserializer(StringDeserializer.class)
            .build();
    assertEquals("localhost:9092", props.get(KafkaConfigs.BOOTSTRAP_SERVERS));
    assertEquals(true, props.get(KafkaConfigs.ENABLE_AUTO_COMMIT));
    assertEquals(AutoOffsetReset.EARLIEST.toString(), props.get(KafkaConfigs.AUTO_OFFSET_RESET));
    assertEquals("test", props.get(KafkaConfigs.GROUP_ID));
    assertEquals(StringDeserializer.class, props.get(KafkaConfigs.KEY_DESERIALIZER));
    assertEquals(StringDeserializer.class, props.get(KafkaConfigs.VALUE_DESERIALIZER));
}
Example #19
Source File: KafkaConfiguration.java    From ad with Apache License 2.0 6 votes vote down vote up 
@Bean
KafkaListenerContainerFactory<ConcurrentMessageListenerContainer<String, String>> kafkaListenerContainerFactory() {
    ConcurrentKafkaListenerContainerFactory<String, String> containerFactory = new ConcurrentKafkaListenerContainerFactory<>();
    containerFactory.setConcurrency(concurrency);

    Map<String, Object> config = Maps.newHashMap();
    config.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    config.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    config.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    // 由于课程原版实现中广告的索引数据是存在于ConcurrentHashMap中, 即每个索引服务实例的jvm中。
    // 所以当每一个索引实例监听kafka topic数据时, 需要保证每个实例都处于不同的消费者组
    // 即各个实例之间需要各不相同的groupId, 保证在部署多实例时, 每个实例都可以加载到完整的索引数据

    // 但在本实现中由于将索引数据单独提出, 存放到了Redis数据库中, 所以应该让所有实例属于同一个消费者组
    // 共同消费kafka topic下的数据, 保证索引数据不会被重复消费。

    // 综上, 若索引数据的存放如果为各个实例自身的jvm, 应该考虑加上以下代码(或自行编写其他实现)保证各实例处于不同的消费者组
    // 若索引数据存放的位置, 是所有检索实例共享的位置, 应该将以下配置取消(或直接删除本类)
    config.put(ConsumerConfig.GROUP_ID_CONFIG, UUID.randomUUID().toString());
    DefaultKafkaConsumerFactory<String, String> consumerFactory = new DefaultKafkaConsumerFactory<String, String>(config);
    containerFactory.setConsumerFactory(consumerFactory);
    return containerFactory;
}
Example #20
Source File: RemoteModuleE2E.java    From flink-statefun with Apache License 2.0 5 votes vote down vote up 
private Consumer<String, InvokeResult> kafkaInvokeResultsConsumer(String bootstrapServers) {
  Properties consumerProps = new Properties();
  consumerProps.setProperty("bootstrap.servers", bootstrapServers);
  consumerProps.setProperty("group.id", "remote-module-e2e");
  consumerProps.setProperty("auto.offset.reset", "earliest");

  KafkaConsumer<String, InvokeResult> consumer =
      new KafkaConsumer<>(
          consumerProps,
          new StringDeserializer(),
          new KafkaProtobufSerializer<>(InvokeResult.parser()));
  consumer.subscribe(Collections.singletonList(INVOKE_RESULTS_TOPIC));

  return consumer;
}
Example #21
Source File: CheckBeginingOffset.java    From BigData-In-Practice with Apache License 2.0 5 votes vote down vote up 
public static KafkaConsumer<String, String> createNewConsumer() {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.0.101:9092");
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "CheckBeginingOffset");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    return new KafkaConsumer<>(props);
}
Example #22
Source File: KafkaConsumerConfig.java    From personal_book_library_web_project with MIT License 5 votes vote down vote up 
public ConsumerFactory<String, MailContext> mailContextConsumerFactory() {

      Map<String, Object> props = new HashMap<>();
      props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, environment.getProperty("kafka.bootstrap.address"));
      props.put(ConsumerConfig.GROUP_ID_CONFIG, environment.getProperty("consumer.group.name"));
      
      return new DefaultKafkaConsumerFactory<>(props, new StringDeserializer(), new JsonDeserializer<>(MailContext.class));
  }
Example #23
Source File: KafkaConfiguration.java    From cubeai with Apache License 2.0 5 votes vote down vote up 
@Bean
public Map<String, Object> consumerConfigs() {
    Map<String, Object> props = new HashMap<>();
    // list of host:port pairs used for establishing the initial connections to the Kakfa cluster
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokers + ":9092");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
    props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
    props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, 120000);
    props.put(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, 180000);
    // allows a pool of processes to divide the work of consuming and processing records
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "umd");

    return props;
}
Example #24
Source File: KafkaConfiguration.java    From spring-rdbms-cdc-kafka-elasticsearch with Apache License 2.0 5 votes vote down vote up 
@Bean
public Map<String, Object> consumerConfigs() {
  return ImmutableMap.<String, Object>builder()
      .put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers)
      .put(ConsumerConfig.GROUP_ID_CONFIG, groupId)
      .put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class)
      .put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, JsonDeserializer.class)
      .put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
      .put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false")
      .put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "10")
      .build();
}
Example #25
Source File: TestUtil.java    From feast with Apache License 2.0 5 votes vote down vote up 
public static <T> KafkaConsumer<String, T> makeKafkaConsumer(
    String bootstrapServers, String topic, Class<?> valueDeserializer) {
  Properties prop = new Properties();
  prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
  prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
  prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, valueDeserializer);
  prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
  prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
  prop.put(ConsumerConfig.GROUP_ID_CONFIG, "test");

  KafkaConsumer<String, T> consumer = new KafkaConsumer<>(prop);

  consumer.subscribe(ImmutableList.of(topic));
  return consumer;
}
Example #26
Source File: CommitSyncInRebalance.java    From BigData-In-Practice with Apache License 2.0 5 votes vote down vote up 
public static Properties initConfig() {
    Properties props = new Properties();
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList);
    props.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
    return props;
}
Example #27
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 5 votes vote down vote up 
public Map<String, Object> consumerConfigs() {
	Map<String, Object> propsMap = new HashMap<>();
	propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
	propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
	propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);
	propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
	propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
	propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
	propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
	propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
	propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig);// 每个批次获取数
	return propsMap;
}
Example #28
Source File: KafkaConsumerConfiguration.java    From ZTuoExchange_framework with MIT License 5 votes vote down vote up 
public Map<String, Object> consumerConfigs() {
	Map<String, Object> propsMap = new HashMap<>();
	propsMap.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, servers);
	propsMap.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, enableAutoCommit);
	propsMap.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, autoCommitInterval);
	propsMap.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sessionTimeout);
	propsMap.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
	propsMap.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
	propsMap.put(ConsumerConfig.GROUP_ID_CONFIG, groupId);
	propsMap.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, autoOffsetReset);
	propsMap.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecordsConfig);// 每个批次获取数
	return propsMap;
}
Example #29
Source File: ConsumerPropertiesBuilderTest.java    From remoting-kafka-plugin with MIT License 5 votes vote down vote up 
@Test
public void testBuildNoBootstrapServers() {
    try {
        Properties props = new ConsumerPropertiesBuilder()
                .withEnableAutoCommit(false)
                .withAutoOffsetReset(AutoOffsetReset.EARLIEST)
                .withGroupID("test")
                .withKeyDeserializer(StringDeserializer.class)
                .withValueDeserializer(StringDeserializer.class)
                .build();
    } catch (Exception e) {
        assertEquals(RemotingKafkaConfigurationException.class, e.getClass());
        assertEquals("Please provide Kafka consumer bootstrap servers", e.getMessage());
    }
}
Example #30
Source File: CodecEndpoint.java    From quarkus with Apache License 2.0 5 votes vote down vote up 
public static KafkaConsumer<String, Movie> createMovieConsumer() {
    Properties props = new Properties();
    props.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:19092");
    props.put(ConsumerConfig.GROUP_ID_CONFIG, "movie");
    props.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    props.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, MovieDeserializer.class.getName());
    props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
    props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    KafkaConsumer<String, Movie> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Collections.singletonList("movies"));
    return consumer;
}