org.apache.flink.streaming.connectors.kafka.config.StartupMode Java Examples
The following examples show how to use
org.apache.flink.streaming.connectors.kafka.config.StartupMode.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaTableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a generic Kafka {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public KafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #2
Source File: Kafka011TableSourceSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase getExpectedKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka011TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets ); }
Example #3
Source File: Kafka011TableSource.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Creates a Kafka 0.11 {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public Kafka011TableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #4
Source File: Kafka011TableSourceSinkFactory.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka011TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #5
Source File: KafkaDynamicSourceBase.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a generic Kafka {@link StreamTableSource}. * * @param outputDataType Source produced data type * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param decodingFormat Decoding format for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. * @param startupTimestampMillis Startup timestamp for offsets; only relevant when startup * mode is {@link StartupMode#TIMESTAMP}. */ protected KafkaDynamicSourceBase( DataType outputDataType, String topic, Properties properties, DecodingFormat<DeserializationSchema<RowData>> decodingFormat, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { this.outputDataType = Preconditions.checkNotNull( outputDataType, "Produced data type must not be null."); this.topic = Preconditions.checkNotNull(topic, "Topic must not be null."); this.properties = Preconditions.checkNotNull(properties, "Properties must not be null."); this.decodingFormat = Preconditions.checkNotNull( decodingFormat, "Decoding format must not be null."); this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null."); this.specificStartupOffsets = Preconditions.checkNotNull( specificStartupOffsets, "Specific offsets must not be null."); this.startupTimestampMillis = startupTimestampMillis; }
Example #6
Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 6 votes |
/** * Variant of {@link KafkaConsumerTestBase#readSequence(StreamExecutionEnvironment, StartupMode, Map, Long, Properties, String, Map)} to * expect reading from the same start offset and the same value count for all partitions of a single Kafka topic. */ protected void readSequence(final StreamExecutionEnvironment env, final StartupMode startupMode, final Map<KafkaTopicPartition, Long> specificStartupOffsets, final Long startupTimestamp, final Properties cc, final int sourceParallelism, final String topicName, final int valuesCount, final int startFrom) throws Exception { HashMap<Integer, Tuple2<Integer, Integer>> partitionsToValuesCountAndStartOffset = new HashMap<>(); for (int i = 0; i < sourceParallelism; i++) { partitionsToValuesCountAndStartOffset.put(i, new Tuple2<>(valuesCount, startFrom)); } readSequence(env, startupMode, specificStartupOffsets, startupTimestamp, cc, topicName, partitionsToValuesCountAndStartOffset); }
Example #7
Source File: Kafka011TableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a Kafka 0.11 {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. * @param startupTimestampMillis Startup timestamp for offsets; only relevant when startup * mode is {@link StartupMode#TIMESTAMP}. */ public Kafka011TableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #8
Source File: KafkaTableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a generic Kafka {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public KafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #9
Source File: Kafka09TableSourceSinkFactory.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka09TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #10
Source File: Kafka09TableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a Kafka 0.9 {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public Kafka09TableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #11
Source File: Kafka010TableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { return new Kafka010TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #12
Source File: Kafka08TableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a Kafka 0.8 {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public Kafka08TableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #13
Source File: Kafka08TableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka08TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #14
Source File: Kafka08ITCase.java From flink with Apache License 2.0 | 6 votes |
@Test(timeout = 60000) public void testInvalidOffset() throws Exception { final int parallelism = 1; // write 20 messages into topic: final String topic = writeSequence("invalidOffsetTopic", 20, parallelism, 1); // set invalid offset: try (CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient()) { ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234); } // read from topic final int valuesCount = 20; final int startFrom = 0; final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); readSequence(env, StartupMode.GROUP_OFFSETS, null, null, standardProps, parallelism, topic, valuesCount, startFrom); deleteTestTopic(topic); }
Example #15
Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 6 votes |
protected void setKafkaConsumerOffset(final StartupMode startupMode, final FlinkKafkaConsumerBase<Tuple2<Integer, Integer>> consumer, final Map<KafkaTopicPartition, Long> specificStartupOffsets, final Long startupTimestamp) { switch (startupMode) { case EARLIEST: consumer.setStartFromEarliest(); break; case LATEST: consumer.setStartFromLatest(); break; case SPECIFIC_OFFSETS: consumer.setStartFromSpecificOffsets(specificStartupOffsets); break; case GROUP_OFFSETS: consumer.setStartFromGroupOffsets(); break; case TIMESTAMP: consumer.setStartFromTimestamp(startupTimestamp); break; } }
Example #16
Source File: Kafka09TableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka09TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #17
Source File: KafkaTableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new KafkaTableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #18
Source File: KafkaTableSourceSinkFactoryTest.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase getExpectedKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new KafkaTableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #19
Source File: Kafka011TableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka011TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #20
Source File: Kafka011TableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a Kafka 0.11 {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public Kafka011TableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #21
Source File: KafkaTableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { return new KafkaTableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #22
Source File: KafkaDynamicSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a generic Kafka {@link StreamTableSource}. * * @param outputDataType Source output data type * @param topic Kafka topic to consume * @param properties Properties for the Kafka consumer * @param decodingFormat Decoding format for decoding records from Kafka * @param startupMode Startup mode for the contained consumer * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS} */ public KafkaDynamicSource( DataType outputDataType, String topic, Properties properties, DecodingFormat<DeserializationSchema<RowData>> decodingFormat, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { super( outputDataType, topic, properties, decodingFormat, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #23
Source File: Kafka010TableSource.java From flink with Apache License 2.0 | 6 votes |
/** * Creates a Kafka 0.10 {@link StreamTableSource}. * * @param schema Schema of the produced table. * @param proctimeAttribute Field name of the processing time attribute. * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute * @param fieldMapping Mapping for the fields of the table schema to * fields of the physical returned type. * @param topic Kafka topic to consume. * @param properties Properties for the Kafka consumer. * @param deserializationSchema Deserialization schema for decoding records from Kafka. * @param startupMode Startup mode for the contained consumer. * @param specificStartupOffsets Specific startup offsets; only relevant when startup * mode is {@link StartupMode#SPECIFIC_OFFSETS}. */ public Kafka010TableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { super( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets); }
Example #24
Source File: Kafka010TableSourceSinkFactoryTest.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase getExpectedKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka010TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets ); }
Example #25
Source File: Kafka010ConnectorDescriptor.java From alchemy with Apache License 2.0 | 6 votes |
@Override KafkaTableSourceBase newTableSource(TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new Kafka010TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets ); }
Example #26
Source File: KafkaConnectorDescriptor.java From alchemy with Apache License 2.0 | 6 votes |
@Override KafkaTableSourceBase newTableSource(TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Optional<Map<String, String>> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets) { return new KafkaTableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, fieldMapping, topic, properties, deserializationSchema, startupMode, specificStartupOffsets ); }
Example #27
Source File: Kafka010DynamicTableFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaDynamicSourceBase createKafkaTableSource( DataType producedDataType, String topic, Properties properties, DecodingFormat<DeserializationSchema<RowData>> decodingFormat, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { return new Kafka010DynamicSource( producedDataType, topic, properties, decodingFormat, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #28
Source File: Kafka011TableSourceSinkFactory.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaTableSourceBase createKafkaTableSource( TableSchema schema, Optional<String> proctimeAttribute, List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors, Map<String, String> fieldMapping, String topic, Properties properties, DeserializationSchema<Row> deserializationSchema, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestampMillis) { return new Kafka011TableSource( schema, proctimeAttribute, rowtimeAttributeDescriptors, Optional.of(fieldMapping), topic, properties, deserializationSchema, startupMode, specificStartupOffsets, startupTimestampMillis); }
Example #29
Source File: Kafka011DynamicTableFactoryTest.java From flink with Apache License 2.0 | 6 votes |
@Override protected KafkaDynamicSourceBase getExpectedScanSource( DataType producedDataType, String topic, Properties properties, DecodingFormat<DeserializationSchema<RowData>> decodingFormat, StartupMode startupMode, Map<KafkaTopicPartition, Long> specificStartupOffsets, long startupTimestamp) { return new Kafka011DynamicSource( producedDataType, topic, properties, decodingFormat, startupMode, specificStartupOffsets, startupTimestamp); }
Example #30
Source File: KafkaConsumerTestBase.java From flink with Apache License 2.0 | 6 votes |
protected void setKafkaConsumerOffset(final StartupMode startupMode, final FlinkKafkaConsumerBase<Tuple2<Integer, Integer>> consumer, final Map<KafkaTopicPartition, Long> specificStartupOffsets, final Long startupTimestamp) { switch (startupMode) { case EARLIEST: consumer.setStartFromEarliest(); break; case LATEST: consumer.setStartFromLatest(); break; case SPECIFIC_OFFSETS: consumer.setStartFromSpecificOffsets(specificStartupOffsets); break; case GROUP_OFFSETS: consumer.setStartFromGroupOffsets(); break; case TIMESTAMP: consumer.setStartFromTimestamp(startupTimestamp); break; } }