org.apache.flink.table.descriptors.KafkaValidator Java Examples

The following examples show how to use org.apache.flink.table.descriptors.KafkaValidator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

6 votes

private Properties getKafkaProperties(DescriptorProperties descriptorProperties) {
	final Properties kafkaProperties = new Properties();

	if (KafkaValidator.hasConciseKafkaProperties(descriptorProperties)) {
		descriptorProperties.asMap().keySet()
			.stream()
			.filter(key -> key.startsWith(CONNECTOR_PROPERTIES))
			.forEach(key -> {
				final String value = descriptorProperties.getString(key);
				final String subKey = key.substring((CONNECTOR_PROPERTIES + '.').length());
				kafkaProperties.put(subKey, value);
			});
	} else {
		final List<Map<String, String>> propsList = descriptorProperties.getFixedIndexedProperties(
			CONNECTOR_PROPERTIES,
			Arrays.asList(CONNECTOR_PROPERTIES_KEY, CONNECTOR_PROPERTIES_VALUE));
		propsList.forEach(kv -> kafkaProperties.put(
			descriptorProperties.getString(kv.get(CONNECTOR_PROPERTIES_KEY)),
			descriptorProperties.getString(kv.get(CONNECTOR_PROPERTIES_VALUE))
		));
	}
	return kafkaProperties;
}

Example #2

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private StartupOptions getStartupOptions(
		DescriptorProperties descriptorProperties,
		String topic) {
	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	final StartupMode startupMode = descriptorProperties
		.getOptionalString(CONNECTOR_STARTUP_MODE)
		.map(modeString -> {
			switch (modeString) {
				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_EARLIEST:
					return StartupMode.EARLIEST;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_LATEST:
					return StartupMode.LATEST;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS:
					return StartupMode.GROUP_OFFSETS;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS:
					buildSpecificOffsets(descriptorProperties, topic, specificOffsets);
					return StartupMode.SPECIFIC_OFFSETS;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_TIMESTAMP:
					return StartupMode.TIMESTAMP;

				default:
					throw new TableException("Unsupported startup mode. Validator should have checked that.");
			}
		}).orElse(StartupMode.GROUP_OFFSETS);
	final StartupOptions options = new StartupOptions();
	options.startupMode = startupMode;
	options.specificOffsets = specificOffsets;
	if (startupMode == StartupMode.TIMESTAMP) {
		options.startupTimestampMillis = descriptorProperties.getLong(CONNECTOR_STARTUP_TIMESTAMP_MILLIS);
	}
	return options;
}

Example #3

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	// allow Kafka timestamps to be used, watermarks can not be received from source
	new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties);
	new KafkaValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #4

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	// allow Kafka timestamps to be used, watermarks can not be received from source
	new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties);
	new KafkaValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #5

Source File: KafkaTableSourceSinkFactoryBase.java From flink with Apache License 2.0

5 votes

private StartupOptions getStartupOptions(
		DescriptorProperties descriptorProperties,
		String topic) {
	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	final StartupMode startupMode = descriptorProperties
		.getOptionalString(CONNECTOR_STARTUP_MODE)
		.map(modeString -> {
			switch (modeString) {
				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_EARLIEST:
					return StartupMode.EARLIEST;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_LATEST:
					return StartupMode.LATEST;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS:
					return StartupMode.GROUP_OFFSETS;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS:
					final List<Map<String, String>> offsetList = descriptorProperties.getFixedIndexedProperties(
						CONNECTOR_SPECIFIC_OFFSETS,
						Arrays.asList(CONNECTOR_SPECIFIC_OFFSETS_PARTITION, CONNECTOR_SPECIFIC_OFFSETS_OFFSET));
					offsetList.forEach(kv -> {
						final int partition = descriptorProperties.getInt(kv.get(CONNECTOR_SPECIFIC_OFFSETS_PARTITION));
						final long offset = descriptorProperties.getLong(kv.get(CONNECTOR_SPECIFIC_OFFSETS_OFFSET));
						final KafkaTopicPartition topicPartition = new KafkaTopicPartition(topic, partition);
						specificOffsets.put(topicPartition, offset);
					});
					return StartupMode.SPECIFIC_OFFSETS;
				default:
					throw new TableException("Unsupported startup mode. Validator should have checked that.");
			}
		}).orElse(StartupMode.GROUP_OFFSETS);
	final StartupOptions options = new StartupOptions();
	options.startupMode = startupMode;
	options.specificOffsets = specificOffsets;
	return options;
}

Example #6

Source File: KafkaTableSourceSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

private DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	// allow Kafka timestamps to be used, watermarks can not be received from source
	new SchemaValidator(true, supportsKafkaTimestamps(), false).validate(descriptorProperties);
	new KafkaValidator().validate(descriptorProperties);

	return descriptorProperties;
}

Example #7

Source File: KafkaTableSourceSinkFactoryBase.java From Flink-CEPplus with Apache License 2.0

5 votes

private StartupOptions getStartupOptions(
		DescriptorProperties descriptorProperties,
		String topic) {
	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	final StartupMode startupMode = descriptorProperties
		.getOptionalString(CONNECTOR_STARTUP_MODE)
		.map(modeString -> {
			switch (modeString) {
				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_EARLIEST:
					return StartupMode.EARLIEST;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_LATEST:
					return StartupMode.LATEST;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_GROUP_OFFSETS:
					return StartupMode.GROUP_OFFSETS;

				case KafkaValidator.CONNECTOR_STARTUP_MODE_VALUE_SPECIFIC_OFFSETS:
					final List<Map<String, String>> offsetList = descriptorProperties.getFixedIndexedProperties(
						CONNECTOR_SPECIFIC_OFFSETS,
						Arrays.asList(CONNECTOR_SPECIFIC_OFFSETS_PARTITION, CONNECTOR_SPECIFIC_OFFSETS_OFFSET));
					offsetList.forEach(kv -> {
						final int partition = descriptorProperties.getInt(kv.get(CONNECTOR_SPECIFIC_OFFSETS_PARTITION));
						final long offset = descriptorProperties.getLong(kv.get(CONNECTOR_SPECIFIC_OFFSETS_OFFSET));
						final KafkaTopicPartition topicPartition = new KafkaTopicPartition(topic, partition);
						specificOffsets.put(topicPartition, offset);
					});
					return StartupMode.SPECIFIC_OFFSETS;
				default:
					throw new TableException("Unsupported startup mode. Validator should have checked that.");
			}
		}).orElse(StartupMode.GROUP_OFFSETS);
	final StartupOptions options = new StartupOptions();
	options.startupMode = startupMode;
	options.specificOffsets = specificOffsets;
	return options;
}

Example #8

Source File: Kafka010TableSourceSinkFactory.java From flink with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #9

Source File: Kafka011TableSourceSinkFactory.java From flink with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #10

Source File: Kafka010TableSourceSinkFactoryTest.java From flink with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #11

Source File: KafkaTableSourceSinkFactory.java From flink with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL;
}

Example #12

Source File: KafkaTableSourceSinkFactoryTest.java From flink with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL;
}

Example #13

Source File: KafkaTableITCase.java From flink with Apache License 2.0

4 votes

@Override
public String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL;
}

Example #14

Source File: Kafka011TableSourceSinkFactory.java From flink with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #15

Source File: Kafka011TableSourceSinkFactoryTest.java From flink with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #16

Source File: Kafka011TableITCase.java From flink with Apache License 2.0

4 votes

@Override
public String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #17

Source File: Kafka010TableSourceSinkFactory.java From flink with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #18

Source File: Kafka010TableITCase.java From flink with Apache License 2.0

4 votes

@Override
public String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #19

Source File: Kafka010TableSourceSinkFactoryTest.java From flink with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #20

Source File: Kafka010TableSourceSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #21

Source File: Kafka09TableSourceSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_09;
}

Example #22

Source File: Kafka08TableSourceSinkFactory.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_08;
}

Example #23

Source File: Kafka08TableSourceSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_08;
}

Example #24

Source File: KafkaTableSourceSinkFactory.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL;
}

Example #25

Source File: KafkaTableSourceSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_UNIVERSAL;
}

Example #26

Source File: Kafka011TableSourceSinkFactory.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #27

Source File: Kafka011TableSourceSinkFactoryTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #28

Source File: Kafka010TableSourceSinkFactory.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_010;
}

Example #29

Source File: Kafka011TableSourceSinkFactoryTest.java From flink with Apache License 2.0

4 votes

@Override
protected String getKafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_011;
}

Example #30

Source File: Kafka09TableSourceSinkFactory.java From flink with Apache License 2.0

4 votes

@Override
protected String kafkaVersion() {
	return KafkaValidator.CONNECTOR_VERSION_VALUE_09;
}