org.apache.flink.table.sources.TableSourceValidation Java Examples

/**
 * Stream table source expects 'update-mode' configuration to be passed.
 */
@Test (expected = NoMatchingTableFactoryException.class)
public void testMissingStreamMode() {

    Pravega pravega = new Pravega();
    Stream stream = Stream.of(SCOPE, STREAM);

    pravega.tableSourceReaderBuilder()
            .forStream(stream)
            .withPravegaConfig(PRAVEGA_CONFIG);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(JSON)
            .withSchema(SCHEMA);

    final Map<String, String> propertiesMap = testDesc.toProperties();

    final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    TableSourceValidation.validateTableSource(source, TableSchema.builder()
            .field("name", DataTypes.STRING() )
            .field("age", DataTypes.INT())
            .build());
    fail("update mode configuration validation failed");
}
 

@Test
public void testJdbcCommonProperties() {
	Map<String, String> properties = getBasicProperties();
	properties.put("connector.driver", "org.apache.derby.jdbc.EmbeddedDriver");
	properties.put("connector.username", "user");
	properties.put("connector.password", "pass");

	final StreamTableSource<?> actual = TableFactoryService.find(StreamTableSourceFactory.class, properties)
		.createStreamTableSource(properties);

	final JdbcOptions options = JdbcOptions.builder()
		.setDBUrl("jdbc:derby:memory:mydb")
		.setTableName("mytable")
		.setDriverName("org.apache.derby.jdbc.EmbeddedDriver")
		.setUsername("user")
		.setPassword("pass")
		.build();
	final JdbcTableSource expected = JdbcTableSource.builder()
		.setOptions(options)
		.setSchema(schema)
		.build();

	TableSourceValidation.validateTableSource(expected, schema);
	TableSourceValidation.validateTableSource(actual, schema);
	assertEquals(expected, actual);
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceValidation.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 

@Override
protected void validateTableSource(TableSource<?> tableSource) {
	super.validateTableSource(tableSource);
	validateTimeCharacteristic(TableSourceValidation.hasRowtimeAttribute(tableSource));
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSourceDescriptor() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String procTime = "procTime";
    final String controllerUri = "tcp://localhost:9090";
    final long delay = 3000L;
    final String streamName = "test";
    final String scopeName = "test";

    final TableSchema tableSchema = TableSchema.builder()
            .field(cityName, DataTypes.STRING())
            .field(total, DataTypes.BIGINT())
            .field(eventTime, DataTypes.TIMESTAMP(3))
            .field(procTime, DataTypes.TIMESTAMP(3))
            .build();

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table source using descriptors and table source factory
    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.BIGINT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                                .rowtime(new Rowtime()
                                            .timestampsFromField(eventTime)
                                            .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay))
                                        )
                            .field(procTime, DataTypes.TIMESTAMP(3)).proctime())
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    assertNotNull(actualSource);
    TableSourceValidation.validateTableSource(actualSource, tableSchema);
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSourceDescriptorWithWatermark() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String controllerUri = "tcp://localhost:9090";
    final String streamName = "test";
    final String scopeName = "test";

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table source using descriptors and table source factory
    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withTimestampAssigner(new MyAssigner())
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    final TableSchema tableSchema = TableSchema.builder()
            .field(cityName, DataTypes.STRING())
            .field(total, DataTypes.INT())
            .field(eventTime, DataTypes.TIMESTAMP(3))
            .build();

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.INT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                            .rowtime(new Rowtime()
                                    .timestampsFromSource()
                                    .watermarksFromSource()
                            ))
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    assertNotNull(actualSource);
    TableSourceValidation.validateTableSource(actualSource, tableSchema);
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {
	// prepare parameters for Kafka table source
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(38, 18))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.field(PROC_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, DataTypes.STRING())
			.field(COUNT, DataTypes.DECIMAL(38, 18))
			.field(TIME, DataTypes.TIMESTAMP(3))
			.build().toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets,
		0L);

	TableSourceValidation.validateTableSource(expected, schema);

	// construct table source using descriptors and table source factory
	final Map<String, String> propertiesMap = new HashMap<>();
	propertiesMap.putAll(createKafkaSourceProperties());
	propertiesMap.put("schema.watermark.0.rowtime", EVENT_TIME);
	propertiesMap.put("schema.watermark.0.strategy.expr", WATERMARK_EXPRESSION);
	propertiesMap.put("schema.watermark.0.strategy.data-type", WATERMARK_DATATYPE.toString());
	propertiesMap.put("schema.4.name", COMPUTED_COLUMN_NAME);
	propertiesMap.put("schema.4.data-type", COMPUTED_COLUMN_DATATYPE.toString());
	propertiesMap.put("schema.4.expr", COMPUTED_COLUMN_EXPRESSION);

	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
	// Test commitOnCheckpoints flag should be true when set consumer group.
	assertTrue(((FlinkKafkaConsumerBase) mock.sourceFunction).getEnableCommitOnCheckpoints());
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSourceWithLegacyProperties() {
	// prepare parameters for Kafka table source
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(38, 18))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.field(PROC_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, DataTypes.STRING())
			.field(COUNT, DataTypes.DECIMAL(38, 18))
			.field(TIME, DataTypes.TIMESTAMP(3))
			.build().toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets,
		0L);

	TableSourceValidation.validateTableSource(expected, schema);

	// construct table source using descriptors and table source factory
	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(createKafkaSourceProperties());

	// use legacy properties
	legacyPropertiesMap.remove("connector.specific-offsets");
	legacyPropertiesMap.remove("connector.properties.bootstrap.servers");
	legacyPropertiesMap.remove("connector.properties.group.id");

	// keep compatible with a specified update-mode
	legacyPropertiesMap.put("update-mode", "append");

	// legacy properties for specific-offsets and properties
	legacyPropertiesMap.put("connector.specific-offsets.0.partition", "0");
	legacyPropertiesMap.put("connector.specific-offsets.0.offset", "100");
	legacyPropertiesMap.put("connector.specific-offsets.1.partition", "1");
	legacyPropertiesMap.put("connector.specific-offsets.1.offset", "123");
	legacyPropertiesMap.put("connector.properties.0.key", "bootstrap.servers");
	legacyPropertiesMap.put("connector.properties.0.value", "dummy");
	legacyPropertiesMap.put("connector.properties.1.key", "group.id");
	legacyPropertiesMap.put("connector.properties.1.value", "dummy");

	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, legacyPropertiesMap)
		.createStreamTableSource(legacyPropertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 

@Override
protected void validateTableSource(TableSource<?> tableSource) {
	super.validateTableSource(tableSource);
	validateTimeCharacteristic(TableSourceValidation.hasRowtimeAttribute(tableSource));
}
 

@Override
public RelDataType getRowType(RelDataTypeFactory typeFactory) {
	final FlinkTypeFactory flinkTypeFactory = (FlinkTypeFactory) typeFactory;
	TableSchema tableSchema = lookupResult.getResolvedSchema();
	final DataType[] fieldDataTypes = tableSchema.getFieldDataTypes();
	CatalogBaseTable catalogTable = lookupResult.getTable();
	if (!isStreamingMode
			&& catalogTable instanceof ConnectorCatalogTable
			&& ((ConnectorCatalogTable<?, ?>) catalogTable).getTableSource().isPresent()) {
		// If the table source is bounded, materialize the time attributes to normal TIMESTAMP type.
		// Now for ConnectorCatalogTable, there is no way to
		// deduce if it is bounded in the table environment, so the data types in TableSchema
		// always patched with TimeAttribute.
		// See ConnectorCatalogTable#calculateSourceSchema
		// for details.

		// Remove the patched time attributes type to let the TableSourceTable handle it.
		// We should remove this logic if the isBatch flag in ConnectorCatalogTable is fixed.
		// TODO: Fix FLINK-14844.
		for (int i = 0; i < fieldDataTypes.length; i++) {
			LogicalType lt = fieldDataTypes[i].getLogicalType();
			if (lt instanceof TimestampType
				&& (((TimestampType) lt).getKind() == TimestampKind.PROCTIME
				|| ((TimestampType) lt).getKind() == TimestampKind.ROWTIME)) {
				int precision = ((TimestampType) lt).getPrecision();
				fieldDataTypes[i] = DataTypes.TIMESTAMP(precision);
			}
		}
	}

	// The following block is a workaround to support tables defined by TableEnvironment.connect() and
	// the actual table sources implement DefinedProctimeAttribute/DefinedRowtimeAttributes.
	// It should be removed after we remove DefinedProctimeAttribute/DefinedRowtimeAttributes.
	Optional<TableSource<?>> sourceOpt = findAndCreateTableSource();
	if (isStreamingMode
		&& tableSchema.getTableColumns().stream().noneMatch(TableColumn::isGenerated)
		&& tableSchema.getWatermarkSpecs().isEmpty()
		&& sourceOpt.isPresent()) {
		TableSource<?> source = sourceOpt.get();
		if (TableSourceValidation.hasProctimeAttribute(source)
				|| TableSourceValidation.hasRowtimeAttribute(source)) {
			// If the table is defined by TableEnvironment.connect(), and use the legacy proctime and rowtime
			// descriptors, the TableSchema should fallback to ConnectorCatalogTable#calculateSourceSchema
			tableSchema = ConnectorCatalogTable.calculateSourceSchema(source, false);
		}
	}

	return TableSourceUtil.getSourceRowType(
		flinkTypeFactory,
		tableSchema,
		scala.Option.empty(),
		isStreamingMode);
}
 

/**
 * Subclasses can override this method to add additional checks.
 *
 * @param tableSource tableSource to validate
 */
protected void validateTableSource(TableSource<?> tableSource) {
	TableSourceValidation.validateTableSource(tableSource);
}
 

/**
 * Subclasses can override this method to add additional checks.
 *
 * @param tableSource tableSource to validate
 */
protected void validateTableSource(TableSource<?> tableSource) {
	TableSourceValidation.validateTableSource(tableSource, tableSource.getTableSchema());
}