org.apache.flink.table.descriptors.Rowtime Java Examples

/**
 * Rowtime attribute should be of type TIMESTAMP.
 */
@Test (expected = ValidationException.class)
public void testWrongRowTimeAttributeType() {
    final Schema schema = new Schema()
            .field("name", DataTypes.STRING())
            .field("age", DataTypes.INT()).rowtime(new Rowtime()
                                                            .timestampsFromField("age")
                                                            .watermarksFromStrategy(
                                                                    new BoundedOutOfOrderTimestamps(30000L)));
    Pravega pravega = new Pravega();
    Stream stream = Stream.of(SCOPE, STREAM);
    pravega.tableSourceReaderBuilder()
            .forStream(stream)
            .withPravegaConfig(PRAVEGA_CONFIG);
    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(JSON)
            .withSchema(schema)
            .inAppendMode();
    final Map<String, String> propertiesMap = testDesc.toProperties();
    FlinkPravegaTableFactoryBase tableFactoryBase = new FlinkPravegaStreamTableSourceFactory();
    tableFactoryBase.createFlinkPravegaTableSource(propertiesMap);
    fail("Schema validation failed");
}
 

protected Map<String, String> createKafkaSourceProperties() {
	return new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
			.withFormat(new TestTableFormat())
			.withSchema(
				new Schema()
					.field(FRUIT_NAME, DataTypes.STRING()).from(NAME)
					.field(COUNT, DataTypes.DECIMAL(38, 18)) // no from so it must match with the input
					.field(EVENT_TIME, DataTypes.TIMESTAMP(3)).rowtime(
						new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
					.field(PROC_TIME, DataTypes.TIMESTAMP(3)).proctime())
			.toProperties();
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(
			Rowtime.ROWTIME_WATERMARKS_TYPE,
			Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED);
	map.put(Rowtime.ROWTIME_WATERMARKS_DELAY, String.valueOf(delay));
	return map;
}
 

/**
 * This method is a default implementation that uses java serialization and it is discouraged.
 * All implementation should provide a more specific set of properties.
 */
@Override
public Map<String, String> toProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM);
	properties.put(Rowtime.ROWTIME_TIMESTAMPS_CLASS, this.getClass().getName());
	properties.put(Rowtime.ROWTIME_TIMESTAMPS_SERIALIZED, EncodingUtils.encodeObjectToString(this));
	return properties;
}
 

/**
 * This method is a default implementation that uses java serialization and it is discouraged.
 * All implementation should provide a more specific set of properties.
 */
@Override
public Map<String, String> toProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM);
	properties.put(Rowtime.ROWTIME_WATERMARKS_CLASS, this.getClass().getName());
	properties.put(Rowtime.ROWTIME_WATERMARKS_SERIALIZED, EncodingUtils.encodeObjectToString(this));
	return properties;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(
			Rowtime.ROWTIME_WATERMARKS_TYPE,
			Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING);
	return map;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD);
	map.put(Rowtime.ROWTIME_TIMESTAMPS_FROM, field);
	return map;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_FIELD);
	map.put(Rowtime.ROWTIME_TIMESTAMPS_FROM, field);
	return map;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(
			Rowtime.ROWTIME_WATERMARKS_TYPE,
			Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_ASCENDING);
	return map;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(
			Rowtime.ROWTIME_WATERMARKS_TYPE,
			Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_PERIODIC_BOUNDED);
	map.put(Rowtime.ROWTIME_WATERMARKS_DELAY, String.valueOf(delay));
	return map;
}
 

/**
 * This method is a default implementation that uses java serialization and it is discouraged.
 * All implementation should provide a more specific set of properties.
 */
@Override
public Map<String, String> toProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_CUSTOM);
	properties.put(Rowtime.ROWTIME_WATERMARKS_CLASS, this.getClass().getName());
	properties.put(Rowtime.ROWTIME_WATERMARKS_SERIALIZED, EncodingUtils.encodeObjectToString(this));
	return properties;
}
 

/**
 * This method is a default implementation that uses java serialization and it is discouraged.
 * All implementation should provide a more specific set of properties.
 */
@Override
public Map<String, String> toProperties() {
	Map<String, String> properties = new HashMap<>();
	properties.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_CUSTOM);
	properties.put(Rowtime.ROWTIME_TIMESTAMPS_CLASS, this.getClass().getName());
	properties.put(Rowtime.ROWTIME_TIMESTAMPS_SERIALIZED, EncodingUtils.encodeObjectToString(this));
	return properties;
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSourceDescriptorWithWatermark() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String controllerUri = "tcp://localhost:9090";
    final String streamName = "test";
    final String scopeName = "test";

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table source using descriptors and table source factory
    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withTimestampAssigner(new MyAssigner())
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    final TableSchema tableSchema = TableSchema.builder()
            .field(cityName, DataTypes.STRING())
            .field(total, DataTypes.INT())
            .field(eventTime, DataTypes.TIMESTAMP(3))
            .build();

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.INT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                            .rowtime(new Rowtime()
                                    .timestampsFromSource()
                                    .watermarksFromSource()
                            ))
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    assertNotNull(actualSource);
    TableSourceValidation.validateTableSource(actualSource, tableSchema);
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE);
	return map;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE);
	return map;
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSinkDescriptor() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String procTime = "procTime";
    final String controllerUri = "tcp://localhost:9090";
    final long delay = 3000L;
    final String streamName = "test";
    final String scopeName = "test";

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table sink using descriptors and table sink factory
    Pravega pravega = new Pravega();
    pravega.tableSinkWriterBuilder()
            .withRoutingKeyField(cityName)
            .withWriterMode(PravegaWriterMode.EXACTLY_ONCE)
            .enableWatermark(true)
            .forStream(stream)
            .enableMetrics(true)
            .withPravegaConfig(pravegaConfig);

    final FlinkPravegaTableSourceTest.TestTableDescriptor testDesc = new FlinkPravegaTableSourceTest.TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.BIGINT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                            .rowtime(new Rowtime()
                                    .timestampsFromField(eventTime)
                                    .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay))
                            )
                            .field(procTime, DataTypes.TIMESTAMP(3)).proctime()
            )
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSink<?> sink = TableFactoryService.find(StreamTableSinkFactory.class, propertiesMap)
            .createStreamTableSink(propertiesMap);

    assertNotNull(sink);
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceUtil.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSourceDescriptor() {
    final String cityName = "fruitName";
    final String total = "count";
    final String eventTime = "eventTime";
    final String procTime = "procTime";
    final String controllerUri = "tcp://localhost:9090";
    final long delay = 3000L;
    final String streamName = "test";
    final String scopeName = "test";

    final TableSchema tableSchema = TableSchema.builder()
            .field(cityName, DataTypes.STRING())
            .field(total, DataTypes.BIGINT())
            .field(eventTime, DataTypes.TIMESTAMP(3))
            .field(procTime, DataTypes.TIMESTAMP(3))
            .build();

    Stream stream = Stream.of(scopeName, streamName);
    PravegaConfig pravegaConfig = PravegaConfig.fromDefaults()
            .withControllerURI(URI.create(controllerUri))
            .withDefaultScope(scopeName);

    // construct table source using descriptors and table source factory
    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    final TestTableDescriptor testDesc = new TestTableDescriptor(pravega)
            .withFormat(new Json().failOnMissingField(false))
            .withSchema(
                    new Schema()
                            .field(cityName, DataTypes.STRING())
                            .field(total, DataTypes.BIGINT())
                            .field(eventTime, DataTypes.TIMESTAMP(3))
                                .rowtime(new Rowtime()
                                            .timestampsFromField(eventTime)
                                            .watermarksFromStrategy(new BoundedOutOfOrderTimestamps(delay))
                                        )
                            .field(procTime, DataTypes.TIMESTAMP(3)).proctime())
            .inAppendMode();

    final Map<String, String> propertiesMap = testDesc.toProperties();
    final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);
    assertNotNull(actualSource);
    TableSourceValidation.validateTableSource(actualSource, tableSchema);
}
 

private void testTableSourceStreamingDescriptor(Stream stream, PravegaConfig pravegaConfig) throws Exception {
    final StreamExecutionEnvironment execEnvRead = StreamExecutionEnvironment.getExecutionEnvironment();
    execEnvRead.setParallelism(1);
    execEnvRead.enableCheckpointing(100);
    execEnvRead.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(execEnvRead,
            EnvironmentSettings.newInstance()
                    // watermark is only supported in blink planner
                    .useBlinkPlanner()
                    .inStreamingMode()
                    .build());
    RESULTS.clear();

    // read data from the stream using Table reader
    Schema schema = new Schema()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            .field("accessTime", DataTypes.TIMESTAMP(3)).rowtime(
                    new Rowtime().timestampsFromField("accessTime").watermarksPeriodicBounded(30000L));

    Pravega pravega = new Pravega();
    pravega.tableSourceReaderBuilder()
            .withReaderGroupScope(stream.getScope())
            .forStream(stream)
            .withPravegaConfig(pravegaConfig);

    ConnectTableDescriptor desc = tableEnv.connect(pravega)
            .withFormat(new Json().failOnMissingField(true))
            .withSchema(schema)
            .inAppendMode();

    final Map<String, String> propertiesMap = desc.toProperties();
    final TableSource<?> source = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
            .createStreamTableSource(propertiesMap);

    String tableSourcePath = tableEnv.getCurrentDatabase() + "." + "MyTableRow";

    ConnectorCatalogTable<?, ?> connectorCatalogSourceTable = ConnectorCatalogTable.source(source, false);

    tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get().createTable(
            ObjectPath.fromString(tableSourcePath),
            connectorCatalogSourceTable, false);

    String sqlQuery = "SELECT user, " +
            "TUMBLE_END(accessTime, INTERVAL '5' MINUTE) AS accessTime, " +
            "COUNT(uri) AS cnt " +
            "from MyTableRow GROUP BY " +
            "user, TUMBLE(accessTime, INTERVAL '5' MINUTE)";
    Table result = tableEnv.sqlQuery(sqlQuery);

    DataStream<Tuple2<Boolean, Row>> resultSet = tableEnv.toRetractStream(result, Row.class);
    StringSink2 stringSink = new StringSink2(8);
    resultSet.addSink(stringSink);

    try {
        execEnvRead.execute("ReadRowData");
    } catch (Exception e) {
        if (!(ExceptionUtils.getRootCause(e) instanceof SuccessException)) {
            throw e;
        }
    }

    log.info("results: {}", RESULTS);
    boolean compare = compare(RESULTS, getExpectedResultsAppend());
    assertTrue("Output does not match expected result", compare);
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(Rowtime.ROWTIME_TIMESTAMPS_TYPE, Rowtime.ROWTIME_TIMESTAMPS_TYPE_VALUE_FROM_SOURCE);
	return map;
}
 

@Override
public Map<String, String> toProperties() {
	Map<String, String> map = new HashMap<>();
	map.put(Rowtime.ROWTIME_WATERMARKS_TYPE, Rowtime.ROWTIME_WATERMARKS_TYPE_VALUE_FROM_SOURCE);
	return map;
}
 

@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceValidation.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}