org.apache.flink.table.factories.TableFactoryService Java Examples

@Test
public void testJDBCWithFilter() {
	Map<String, String> properties = getBasicProperties();
	properties.put("connector.driver", "org.apache.derby.jdbc.EmbeddedDriver");
	properties.put("connector.username", "user");
	properties.put("connector.password", "pass");

	final TableSource<?> actual = ((JDBCTableSource) TableFactoryService
		.find(StreamTableSourceFactory.class, properties)
		.createStreamTableSource(properties))
		.projectFields(new int[] {0, 2});

	Map<String, DataType> projectedFields = ((FieldsDataType) actual.getProducedDataType()).getFieldDataTypes();
	assertEquals(projectedFields.get("aaa"), DataTypes.INT());
	assertNull(projectedFields.get("bbb"));
	assertEquals(projectedFields.get("ccc"), DataTypes.DOUBLE());
}
 

@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = new HashMap<>();
	properties.putAll(new Schema().schema(TableSchema.fromTypeInfo(SCHEMA)).toProperties());
	properties.putAll(new Csv().deriveSchema().toProperties());

	final CsvRowSerializationSchema expectedSer = new CsvRowSerializationSchema.Builder(SCHEMA).build();
	final CsvRowDeserializationSchema expectedDeser = new CsvRowDeserializationSchema.Builder(SCHEMA).build();

	final SerializationSchema<?> actualSer = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);

	assertEquals(expectedSer, actualSer);

	final DeserializationSchema<?> actualDeser = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);

	assertEquals(expectedDeser, actualDeser);
}
 

private SerializationSchema<Row> getSerializationSchema(Map<String, String> properties) {
	final String formatType = properties.get(FORMAT_TYPE);
	// we could have added this check to the table factory context
	// but this approach allows to throw more helpful error messages
	// if the supported format has not been added
	if (formatType == null || !formatType.equals(SUPPORTED_FORMAT_TYPE)) {
		throw new ValidationException(
			"The Elasticsearch sink requires a '" + SUPPORTED_FORMAT_TYPE + "' format.");
	}

	@SuppressWarnings("unchecked")
	final SerializationSchemaFactory<Row> formatFactory = TableFactoryService.find(
		SerializationSchemaFactory.class,
		properties,
		this.getClass().getClassLoader());
	return formatFactory.createSerializationSchema(properties);
}
 

@Test
public void testTableSourceCommitOnCheckpointsDisabled() {
	Map<String, String> propertiesMap = new HashMap<>();
	createKafkaSourceProperties().forEach((k, v) -> {
		if (!k.equals("connector.properties.group.id")) {
			propertiesMap.put(k, v);
		}
	});
	final TableSource<?> tableSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	// Test commitOnCheckpoints flag should be false when do not set consumer group.
	((KafkaTableSourceBase) tableSource).getDataStream(mock);
	assertTrue(mock.sourceFunction instanceof FlinkKafkaConsumerBase);
	assertFalse(((FlinkKafkaConsumerBase) mock.sourceFunction).getEnableCommitOnCheckpoints());
}
 

private SerializationSchema<Row> getSerializationSchema(Map<String, String> properties) {
	final String formatType = properties.get(FORMAT_TYPE);
	// we could have added this check to the table factory context
	// but this approach allows to throw more helpful error messages
	// if the supported format has not been added
	if (formatType == null || !formatType.equals(SUPPORTED_FORMAT_TYPE)) {
		throw new ValidationException(
			"The Elasticsearch sink requires a '" + SUPPORTED_FORMAT_TYPE + "' format.");
	}

	@SuppressWarnings("unchecked")
	final SerializationSchemaFactory<Row> formatFactory = TableFactoryService.find(
		SerializationSchemaFactory.class,
		properties,
		this.getClass().getClassLoader());
	return formatFactory.createSerializationSchema(properties);
}
 

@Test
public void testJdbcFieldsProjection() {
	Map<String, String> properties = getBasicProperties();
	properties.put("connector.driver", "org.apache.derby.jdbc.EmbeddedDriver");
	properties.put("connector.username", "user");
	properties.put("connector.password", "pass");

	final TableSource<?> actual = ((JdbcTableSource) TableFactoryService
		.find(StreamTableSourceFactory.class, properties)
		.createStreamTableSource(properties))
		.projectFields(new int[] {0, 2});

	List<DataType> projectedFields = actual.getProducedDataType().getChildren();
	assertEquals(Arrays.asList(DataTypes.INT(), DataTypes.DOUBLE()), projectedFields);

	// test jdbc table source description
	List<String> fieldNames = ((RowType) actual.getProducedDataType().getLogicalType()).getFieldNames();
	String expectedSourceDescription = actual.getClass().getSimpleName()
		+ "(" + String.join(", ", fieldNames.stream().toArray(String[]::new)) + ")";
	assertEquals(expectedSourceDescription, actual.explainSource());
}
 

@Test
public void testJdbcCommonProperties() {
	Map<String, String> properties = getBasicProperties();
	properties.put("connector.driver", "org.apache.derby.jdbc.EmbeddedDriver");
	properties.put("connector.username", "user");
	properties.put("connector.password", "pass");

	final StreamTableSource<?> actual = TableFactoryService.find(StreamTableSourceFactory.class, properties)
		.createStreamTableSource(properties);

	final JdbcOptions options = JdbcOptions.builder()
		.setDBUrl("jdbc:derby:memory:mydb")
		.setTableName("mytable")
		.setDriverName("org.apache.derby.jdbc.EmbeddedDriver")
		.setUsername("user")
		.setPassword("pass")
		.build();
	final JdbcTableSource expected = JdbcTableSource.builder()
		.setOptions(options)
		.setSchema(schema)
		.build();

	TableSourceValidation.validateTableSource(expected, schema);
	TableSourceValidation.validateTableSource(actual, schema);
	assertEquals(expected, actual);
}
 

@Test (expected = ValidationException.class)
public void testMissingFormatDefinition() {
    Pravega pravega = new Pravega();
    Stream stream = Stream.of(SCOPE, STREAM);

    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("name")
            .forStream(stream)
            .withPravegaConfig(PRAVEGA_CONFIG);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withSchema(SCHEMA)
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);
    fail("table factory validation failed");
}
 

@Test (expected = ValidationException.class)
public void testMissingSchemaDefinition() {
    Pravega pravega = new Pravega();
    Stream stream = Stream.of(SCOPE, STREAM);

    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("name")
            .forStream(stream)
            .withPravegaConfig(PRAVEGA_CONFIG);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(JSON)
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);
    fail("missing schema validation failed");
}
 

@Test
public void testValidWriterModeExactlyOnce() {
    Pravega pravega = new Pravega();
    Stream stream = Stream.of(SCOPE, STREAM);

    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("name").withWriterMode(PravegaWriterMode.EXACTLY_ONCE)
            .forStream(stream)
            .withPravegaConfig(PRAVEGA_CONFIG);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(JSON)
            .withSchema(SCHEMA)
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);
    assertNotNull(sink);
}
 

@Test (expected = ValidationException.class)
public void testInvalidWriterMode() {
    Pravega pravega = new Pravega();
    Stream stream = Stream.of(SCOPE, STREAM);

    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("name")
            .forStream(stream)
            .withPravegaConfig(PRAVEGA_CONFIG);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(JSON)
            .withSchema(SCHEMA)
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    Map<String, String> test = new HashMap<>(propertiesMap);
    test.put(CONNECTOR_WRITER_MODE, "foo");
    TableFactoryService.find(StreamTableSinkFactory.class, test)
            .createStreamTableSink(test);
    fail("writer mode validation failed");
}
 

/**
 * For sink, stream name information is mandatory.
 */
@Test (expected = IllegalStateException.class)
public void testMissingStreamNameForWriter() {
    Pravega pravega = new Pravega();

    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField("name");

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(JSON)
            .withSchema(SCHEMA)
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);
    fail("stream name validation failed");
}
 

@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = new HashMap<>();
	properties.putAll(new Schema().schema(TableSchema.fromTypeInfo(SCHEMA)).toProperties());
	properties.putAll(new Csv().deriveSchema().toProperties());

	final CsvRowSerializationSchema expectedSer = new CsvRowSerializationSchema.Builder(SCHEMA).build();
	final CsvRowDeserializationSchema expectedDeser = new CsvRowDeserializationSchema.Builder(SCHEMA).build();

	final SerializationSchema<?> actualSer = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);

	assertEquals(expectedSer, actualSer);

	final DeserializationSchema<?> actualDeser = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);

	assertEquals(expectedDeser, actualDeser);
}
 

@Test
public void testSchemaDerivation() {
	final Map<String, String> properties = new HashMap<>();
	properties.putAll(new Schema().schema(TableSchema.fromTypeInfo(SCHEMA)).toProperties());
	properties.putAll(new Csv().toProperties());

	final CsvRowSerializationSchema expectedSer = new CsvRowSerializationSchema.Builder(SCHEMA).build();
	final CsvRowDeserializationSchema expectedDeser = new CsvRowDeserializationSchema.Builder(SCHEMA).build();

	final SerializationSchema<?> actualSer = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);

	assertEquals(expectedSer, actualSer);

	final DeserializationSchema<?> actualDeser = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);

	assertEquals(expectedDeser, actualDeser);
}
 

protected SerializationSchema<Row> getSerializationSchema(Map<String, String> properties) {
    @SuppressWarnings("unchecked")
    final SerializationSchemaFactory<Row> formatFactory = TableFactoryService.find(
            SerializationSchemaFactory.class,
            properties,
            this.getClass().getClassLoader());
    return formatFactory.createSerializationSchema(properties);
}
 

@Test
public void testTableSinkWithLegacyProperties() {
	// prepare parameters for Elasticsearch table sink
	final TableSchema schema = createTestSchema();

	final ElasticsearchUpsertTableSinkBase expectedSink = getExpectedTableSink(
		false,
		schema,
		Collections.singletonList(new Host(HOSTNAME, PORT, SCHEMA)),
		INDEX,
		DOC_TYPE,
		KEY_DELIMITER,
		KEY_NULL_LITERAL,
		JsonRowSerializationSchema.builder().withTypeInfo(schema.toRowType()).build(),
		XContentType.JSON,
		new DummyFailureHandler(),
		createTestSinkOptions(),
		IndexGeneratorFactory.createIndexGenerator(INDEX, schema));

	// construct table sink using descriptors and table sink factory
	final Map<String, String> elasticSearchProperties = createElasticSearchProperties();

	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(elasticSearchProperties);
	// use legacy properties
	legacyPropertiesMap.remove("connector.hosts");

	legacyPropertiesMap.put("connector.hosts.0.hostname", "host1");
	legacyPropertiesMap.put("connector.hosts.0.port", "1234");
	legacyPropertiesMap.put("connector.hosts.0.protocol", "https");

	final TableSink<?> actualSink = TableFactoryService.find(StreamTableSinkFactory.class, legacyPropertiesMap)
		.createStreamTableSink(legacyPropertiesMap);

	assertEquals(expectedSink, actualSink);
}
 

@Test
public void testSupportedProperties() {
	Map<String, String> properties = getBasicProperties();

	properties.put(CONNECTOR_BATCH_SIZE, "1");
	properties.put(CONNECTOR_BUFFER_SIZE, "1");
	properties.put(CONNECTOR_RETRY_TIMEOUT_IN_MILLS, "3");
	properties.put(CONNECTOR_MAX_RETRY_TIMES, "10");
	properties.put(CONNECTOR_BATCH_WRITE_TIMEOUT_IN_MILLS, "5");

	final TableSink<?> actual = TableFactoryService.find(TableSinkFactory.class, properties)
			.createTableSink(properties);

	assertTrue(actual instanceof DatahubTableSink);
}
 

private void testSchemaDeserializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final JsonRowDeserializationSchema expected2 = new JsonRowDeserializationSchema.Builder(SCHEMA).build();
	assertEquals(expected2, actual2);
}
 

private void testAvroSchemaDeserializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final AvroRowDeserializationSchema expected2 = new AvroRowDeserializationSchema(AVRO_SCHEMA);
	assertEquals(expected2, actual2);
}
 

@Test
public void testJdbcReadProperties() {
	Map<String, String> properties = getBasicProperties();
	properties.put("connector.read.query", "SELECT aaa FROM mytable");
	properties.put("connector.read.partition.column", "aaa");
	properties.put("connector.read.partition.lower-bound", "-10");
	properties.put("connector.read.partition.upper-bound", "100");
	properties.put("connector.read.partition.num", "10");
	properties.put("connector.read.fetch-size", "20");

	final StreamTableSource<?> actual = TableFactoryService.find(StreamTableSourceFactory.class, properties)
		.createStreamTableSource(properties);

	final JdbcOptions options = JdbcOptions.builder()
		.setDBUrl("jdbc:derby:memory:mydb")
		.setTableName("mytable")
		.build();
	final JdbcReadOptions readOptions = JdbcReadOptions.builder()
		.setQuery("SELECT aaa FROM mytable")
		.setPartitionColumnName("aaa")
		.setPartitionLowerBound(-10)
		.setPartitionUpperBound(100)
		.setNumPartitions(10)
		.setFetchSize(20)
		.build();
	final JdbcTableSource expected = JdbcTableSource.builder()
		.setOptions(options)
		.setReadOptions(readOptions)
		.setSchema(schema)
		.build();

	assertEquals(expected, actual);
}
 

private void testJsonSchemaDeserializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final JsonRowDeserializationSchema expected2 = new JsonRowDeserializationSchema.Builder(JSON_SCHEMA)
		.failOnMissingField()
		.build();
	assertEquals(expected2, actual2);
}
 

private void testJsonSchemaDeserializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final JsonRowDeserializationSchema expected2 = new JsonRowDeserializationSchema.Builder(JSON_SCHEMA)
		.failOnMissingField()
		.build();
	assertEquals(expected2, actual2);
}
 

private void testRecordClassDeserializationSchema(Map<String, String> properties) {
	final SerializationSchema<?> actual1 = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);
	final SerializationSchema<?> expected1 = new AvroRowSerializationSchema(AVRO_SPECIFIC_RECORD);
	assertEquals(expected1, actual1);
}
 

@Test
public void testDisableQuoteCharacter() {
	final Map<String, String> properties = new Csv()
		.schema(SCHEMA)
		.fieldDelimiter(';')
		.lineDelimiter("\r\n")
		.allowComments()
		.ignoreParseErrors()
		.arrayElementDelimiter("|")
		.escapeCharacter('\\')
		.nullLiteral("n/a")
		.disableQuoteCharacter()
		.toProperties();

	final CsvRowSerializationSchema expectedSer = new CsvRowSerializationSchema.Builder(SCHEMA)
		.setFieldDelimiter(';')
		.setLineDelimiter("\r\n")
		.setArrayElementDelimiter("|")
		.setEscapeCharacter('\\')
		.setNullLiteral("n/a")
		.disableQuoteCharacter()
		.build();

	final SerializationSchema<?> actualSer = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);

	assertEquals(expectedSer, actualSer);
}
 

private void testAvroSchemaSerializationSchema(Map<String, String> properties) {
	final SerializationSchema<?> actual1 = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);
	final SerializationSchema<?> expected1 = new AvroRowSerializationSchema(AVRO_SCHEMA);
	assertEquals(expected1, actual1);
}
 

private void testAvroSchemaDeserializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final AvroRowDeserializationSchema expected2 = new AvroRowDeserializationSchema(AVRO_SCHEMA);
	assertEquals(expected2, actual2);
}
 

private void testRecordClassDeserializationSchema(Map<String, String> properties) {
	final SerializationSchema<?> actual1 = TableFactoryService
		.find(SerializationSchemaFactory.class, properties)
		.createSerializationSchema(properties);
	final SerializationSchema<?> expected1 = new AvroRowSerializationSchema(AVRO_SPECIFIC_RECORD);
	assertEquals(expected1, actual1);
}
 

private void testRecordClassSerializationSchema(Map<String, String> properties) {
	final DeserializationSchema<?> actual2 = TableFactoryService
		.find(DeserializationSchemaFactory.class, properties)
		.createDeserializationSchema(properties);
	final AvroRowDeserializationSchema expected2 = new AvroRowDeserializationSchema(AVRO_SPECIFIC_RECORD);
	assertEquals(expected2, actual2);
}
 

@Test
public void testDisallowEmbedded() {
	expectedException.expect(IllegalArgumentException.class);
	final Map<String, String> properties = new HiveCatalogDescriptor().toProperties();

	TableFactoryService.find(CatalogFactory.class, properties).createCatalog("my_catalog", properties);
}
 

private SerializationSchema<Row> getSerializationSchema(Map<String, String> properties) {
	@SuppressWarnings("unchecked")
	final SerializationSchemaFactory<Row> formatFactory = TableFactoryService.find(
		SerializationSchemaFactory.class,
		properties,
		this.getClass().getClassLoader());
	return formatFactory.createSerializationSchema(properties);
}