Java Code Examples for org.apache.flink.streaming.connectors.kafka.config.StartupMode

The following examples show how to use org.apache.flink.streaming.connectors.kafka.config.StartupMode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka09TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: Kafka09TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.9 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka09TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 3
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka09TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets
	);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: Kafka08TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.8 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka08TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 5
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka08TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 6
Source Project: Flink-CEPplus   Source File: Kafka08ITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testInvalidOffset() throws Exception {

	final int parallelism = 1;

	// write 20 messages into topic:
	final String topic = writeSequence("invalidOffsetTopic", 20, parallelism, 1);

	// set invalid offset:
	try (CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient()) {
		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
	}

	// read from topic
	final int valuesCount = 20;
	final int startFrom = 0;

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	readSequence(env, StartupMode.GROUP_OFFSETS, null, null, standardProps, parallelism, topic, valuesCount, startFrom);

	deleteTestTopic(topic);
}
 
Example 7
Source Project: flink   Source File: KafkaDynamicTableFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaDynamicSourceBase getExpectedScanSource(
		DataType producedDataType,
		String topic,
		Properties properties,
		DecodingFormat<DeserializationSchema<RowData>> decodingFormat,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestamp) {
	return new KafkaDynamicSource(
			producedDataType,
			topic,
			properties,
			decodingFormat,
			startupMode,
			specificStartupOffsets,
			startupTimestamp);
}
 
Example 8
Source Project: Flink-CEPplus   Source File: KafkaTableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public KafkaTableSource(
	TableSchema schema,
	Optional<String> proctimeAttribute,
	List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
	Optional<Map<String, String>> fieldMapping,
	String topic,
	Properties properties,
	DeserializationSchema<Row> deserializationSchema,
	StartupMode startupMode,
	Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 9
Source Project: flink   Source File: Kafka010TableSourceSinkFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {

	return new Kafka010TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis
	);
}
 
Example 10
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
	TableSchema schema,
	Optional<String> proctimeAttribute,
	List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
	Map<String, String> fieldMapping,
	String topic,
	Properties properties,
	DeserializationSchema<Row> deserializationSchema,
	StartupMode startupMode,
	Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new KafkaTableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 11
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka011TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 12
Source Project: Flink-CEPplus   Source File: Kafka011TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.11 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka011TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 13
@Override
protected KafkaTableSourceBase getExpectedKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka011TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets
	);
}
 
Example 14
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka010TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: Kafka010TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.10 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka010TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 16
Source Project: Flink-CEPplus   Source File: KafkaTableSourceBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                Schema of the produced table.
 * @param topic                 Kafka topic to consume.
 * @param properties            Properties for the Kafka consumer.
 * @param deserializationSchema Deserialization schema for decoding records from Kafka.
 */
protected KafkaTableSourceBase(
		TableSchema schema,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema) {
	this(
		schema,
		Optional.empty(),
		Collections.emptyList(),
		Optional.empty(),
		topic, properties,
		deserializationSchema,
		StartupMode.GROUP_OFFSETS,
		Collections.emptyMap());
}
 
Example 17
Source Project: flink   Source File: Kafka011DynamicTableFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaDynamicSourceBase createKafkaTableSource(
		DataType producedDataType,
		String topic,
		Properties properties,
		DecodingFormat<DeserializationSchema<RowData>> decodingFormat,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {
	return new Kafka011DynamicSource(
			producedDataType,
			topic,
			properties,
			decodingFormat,
			startupMode,
			specificStartupOffsets,
			startupTimestampMillis);
}
 
Example 18
Source Project: flink   Source File: Kafka011DynamicSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.11 {@link org.apache.flink.table.connector.source.ScanTableSource}.
 *
 * @param outputDataType         Source output data type
 * @param topic                  Kafka topic to consume
 * @param properties             Properties for the Kafka consumer
 * @param decodingFormat         Decoding format for decoding records from Kafka
 * @param startupMode            Startup mode for the contained consumer
 * @param specificStartupOffsets Specific startup offsets; only relevant when startup
 *                               mode is {@link StartupMode#SPECIFIC_OFFSETS}
 * @param startupTimestampMillis Startup timestamp for offsets; only relevant when startup
 *                               mode is {@link StartupMode#TIMESTAMP}
 */
public Kafka011DynamicSource(
		DataType outputDataType,
		String topic,
		Properties properties,
		DecodingFormat<DeserializationSchema<RowData>> decodingFormat,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {

	super(
			outputDataType,
			topic,
			properties,
			decodingFormat,
			startupMode,
			specificStartupOffsets,
			startupTimestampMillis);
}
 
Example 19
Source Project: flink   Source File: KafkaDynamicSourceBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param outputDataType         Source produced data type
 * @param topic                  Kafka topic to consume.
 * @param properties             Properties for the Kafka consumer.
 * @param decodingFormat         Decoding format for decoding records from Kafka.
 * @param startupMode            Startup mode for the contained consumer.
 * @param specificStartupOffsets Specific startup offsets; only relevant when startup
 *                               mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 * @param startupTimestampMillis Startup timestamp for offsets; only relevant when startup
 *                               mode is {@link StartupMode#TIMESTAMP}.
 */
protected KafkaDynamicSourceBase(
		DataType outputDataType,
		String topic,
		Properties properties,
		DecodingFormat<DeserializationSchema<RowData>> decodingFormat,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {
	this.outputDataType = Preconditions.checkNotNull(
			outputDataType, "Produced data type must not be null.");
	this.topic = Preconditions.checkNotNull(topic, "Topic must not be null.");
	this.properties = Preconditions.checkNotNull(properties, "Properties must not be null.");
	this.decodingFormat = Preconditions.checkNotNull(
		decodingFormat, "Decoding format must not be null.");
	this.startupMode = Preconditions.checkNotNull(startupMode, "Startup mode must not be null.");
	this.specificStartupOffsets = Preconditions.checkNotNull(
		specificStartupOffsets, "Specific offsets must not be null.");
	this.startupTimestampMillis = startupTimestampMillis;
}
 
Example 20
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Variant of {@link KafkaConsumerTestBase#readSequence(StreamExecutionEnvironment, StartupMode, Map, Long, Properties, String, Map)} to
 * expect reading from the same start offset and the same value count for all partitions of a single Kafka topic.
 */
protected void readSequence(final StreamExecutionEnvironment env,
							final StartupMode startupMode,
							final Map<KafkaTopicPartition, Long> specificStartupOffsets,
							final Long startupTimestamp,
							final Properties cc,
							final int sourceParallelism,
							final String topicName,
							final int valuesCount,
							final int startFrom) throws Exception {
	HashMap<Integer, Tuple2<Integer, Integer>> partitionsToValuesCountAndStartOffset = new HashMap<>();
	for (int i = 0; i < sourceParallelism; i++) {
		partitionsToValuesCountAndStartOffset.put(i, new Tuple2<>(valuesCount, startFrom));
	}
	readSequence(env, startupMode, specificStartupOffsets, startupTimestamp, cc, topicName, partitionsToValuesCountAndStartOffset);
}
 
Example 21
Source Project: flink   Source File: Kafka011TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.11 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 * @param startupTimestampMillis	  Startup timestamp for offsets; only relevant when startup
 *                                    mode is {@link StartupMode#TIMESTAMP}.
 */
public Kafka011TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis);
}
 
Example 22
Source Project: flink   Source File: KafkaTableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public KafkaTableSource(
	TableSchema schema,
	Optional<String> proctimeAttribute,
	List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
	Optional<Map<String, String>> fieldMapping,
	String topic,
	Properties properties,
	DeserializationSchema<Row> deserializationSchema,
	StartupMode startupMode,
	Map<KafkaTopicPartition, Long> specificStartupOffsets,
	long startupTimestampMillis) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis);
}
 
Example 23
Source Project: flink   Source File: Kafka09TableSourceSinkFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka09TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 24
Source Project: flink   Source File: Kafka09TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.9 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka09TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 25
Source Project: flink   Source File: Kafka010TableSourceSinkFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets,
		long startupTimestampMillis) {

	return new Kafka010TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets,
		startupTimestampMillis);
}
 
Example 26
Source Project: flink   Source File: Kafka08TableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a Kafka 0.8 {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public Kafka08TableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Optional<Map<String, String>> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 27
Source Project: flink   Source File: Kafka08TableSourceSinkFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected KafkaTableSourceBase createKafkaTableSource(
		TableSchema schema,
		Optional<String> proctimeAttribute,
		List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
		Map<String, String> fieldMapping,
		String topic,
		Properties properties,
		DeserializationSchema<Row> deserializationSchema,
		StartupMode startupMode,
		Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	return new Kafka08TableSource(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		Optional.of(fieldMapping),
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}
 
Example 28
Source Project: flink   Source File: Kafka08ITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testInvalidOffset() throws Exception {

	final int parallelism = 1;

	// write 20 messages into topic:
	final String topic = writeSequence("invalidOffsetTopic", 20, parallelism, 1);

	// set invalid offset:
	try (CuratorFramework curatorClient = ((KafkaTestEnvironmentImpl) kafkaServer).createCuratorClient()) {
		ZookeeperOffsetHandler.setOffsetInZooKeeper(curatorClient, standardProps.getProperty("group.id"), topic, 0, 1234);
	}

	// read from topic
	final int valuesCount = 20;
	final int startFrom = 0;

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	readSequence(env, StartupMode.GROUP_OFFSETS, null, null, standardProps, parallelism, topic, valuesCount, startFrom);

	deleteTestTopic(topic);
}
 
Example 29
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected void setKafkaConsumerOffset(final StartupMode startupMode,
									final FlinkKafkaConsumerBase<Tuple2<Integer, Integer>> consumer,
									final Map<KafkaTopicPartition, Long> specificStartupOffsets,
									final Long startupTimestamp) {
	switch (startupMode) {
		case EARLIEST:
			consumer.setStartFromEarliest();
			break;
		case LATEST:
			consumer.setStartFromLatest();
			break;
		case SPECIFIC_OFFSETS:
			consumer.setStartFromSpecificOffsets(specificStartupOffsets);
			break;
		case GROUP_OFFSETS:
			consumer.setStartFromGroupOffsets();
			break;
		case TIMESTAMP:
			consumer.setStartFromTimestamp(startupTimestamp);
			break;
	}
}
 
Example 30
Source Project: flink   Source File: KafkaTableSource.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a generic Kafka {@link StreamTableSource}.
 *
 * @param schema                      Schema of the produced table.
 * @param proctimeAttribute           Field name of the processing time attribute.
 * @param rowtimeAttributeDescriptors Descriptor for a rowtime attribute
 * @param fieldMapping                Mapping for the fields of the table schema to
 *                                    fields of the physical returned type.
 * @param topic                       Kafka topic to consume.
 * @param properties                  Properties for the Kafka consumer.
 * @param deserializationSchema       Deserialization schema for decoding records from Kafka.
 * @param startupMode                 Startup mode for the contained consumer.
 * @param specificStartupOffsets      Specific startup offsets; only relevant when startup
 *                                    mode is {@link StartupMode#SPECIFIC_OFFSETS}.
 */
public KafkaTableSource(
	TableSchema schema,
	Optional<String> proctimeAttribute,
	List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors,
	Optional<Map<String, String>> fieldMapping,
	String topic,
	Properties properties,
	DeserializationSchema<Row> deserializationSchema,
	StartupMode startupMode,
	Map<KafkaTopicPartition, Long> specificStartupOffsets) {

	super(
		schema,
		proctimeAttribute,
		rowtimeAttributeDescriptors,
		fieldMapping,
		topic,
		properties,
		deserializationSchema,
		startupMode,
		specificStartupOffsets);
}