Java Code Examples for org.apache.flink.table.sources.tsextractors.ExistingField

The following examples show how to use org.apache.flink.table.sources.tsextractors.ExistingField. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: alchemy   Source File: SourceDescriptorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void buildKafkaSource() throws Exception {
    File file = ResourceUtils.getFile("classpath:yaml/kafka-source.yaml");
    SourceDescriptor sourceDescriptor = BindPropertiesUtil.bindProperties(file, SourceDescriptor.class);
    Kafka010ConnectorDescriptor connectorDescriptor = BindPropertiesUtil.bindProperties(sourceDescriptor.getConnector(), Kafka010ConnectorDescriptor.class);
    assertThat(connectorDescriptor.getTopic()).isEqualTo("app-log");
    assertThat(connectorDescriptor.getStartupMode()).isEqualTo("earliest-offset");
    assertThat(connectorDescriptor.getSpecificOffsets().get("1")).isEqualTo("1000");
    assertThat(connectorDescriptor.getSpecificOffsets().get("2")).isEqualTo("3000");
    assertThat(connectorDescriptor.getProperties().get("bootstrap.servers")).isEqualTo("127.0.0.1:9092");
    assertThat(connectorDescriptor.getProperties().get("group.id")).isEqualTo("testGroup");
    assertThat(sourceDescriptor.getSchema()).isNotNull();
    FormatDescriptor formatDescriptor = sourceDescriptor.getFormat();
    KafkaTableSourceBase alchemyKafkaTableSource = connectorDescriptor.buildSource(sourceDescriptor.getSchema(), formatDescriptor);
    assertThat(alchemyKafkaTableSource).isNotNull();
    assertThat(alchemyKafkaTableSource.getProctimeAttribute()).isEqualTo("procTime");
    List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = alchemyKafkaTableSource.getRowtimeAttributeDescriptors();
    assertThat(rowtimeAttributeDescriptors).isNotNull();
    assertThat(rowtimeAttributeDescriptors.get(0).getAttributeName()).isEqualTo("rowTime");
    assertThat(rowtimeAttributeDescriptors.get(0).getTimestampExtractor()).isInstanceOf(ExistingField.class);
    assertThat(rowtimeAttributeDescriptors.get(0).getWatermarkStrategy()).isInstanceOf(BoundedOutOfOrderTimestamps.class);
    DeserializationSchema deserializationSchema = formatDescriptor.transform(new Tuple2<>(alchemyKafkaTableSource.getReturnType(), true));
    assertThat(deserializationSchema).isInstanceOf(GrokRowDeserializationSchema.class);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: StreamSQLTestProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"rowtime",
			new ExistingField("ts"),
			new BoundedOutOfOrderTimestamps(100)));
}
 
Example 3
Source Project: flink   Source File: UnboundedTransactionTableSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"timestamp",
			new ExistingField("timestamp"),
			new BoundedOutOfOrderTimestamps(100)));
}
 
Example 4
Source Project: flink   Source File: StreamSQLTestProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"rowtime",
			new ExistingField("ts"),
			new BoundedOutOfOrderTimestamps(100)));
}
 
Example 5
Source Project: alchemy   Source File: Timestamps.java    License: Apache License 2.0 5 votes vote down vote up
public TimestampExtractor get(){
    if(type == null){
        return null;
    }
    if(type.equals(Type.FIELD.getType())){
        return new ExistingField(from);
    }else if(type.equals(Type.SOURCE.getType())){
        return new StreamRecordTimestamp();
    }
    return null;
}
 
Example 6
Source Project: pravega-samples   Source File: PopularTaxiVendor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handleRequest() {

    TableSchema tableSchema = TripRecord.getTableSchema();

    FlinkPravegaJsonTableSource source = FlinkPravegaJsonTableSource.builder()
            .forStream(Stream.of(getScope(), getStream()).getScopedName())
            .withPravegaConfig(getPravegaConfig())
            .failOnMissingField(true)
            .withRowtimeAttribute("pickupTime", new ExistingField("pickupTime"), new BoundedOutOfOrderTimestamps(30000L))
            .withSchema(tableSchema)
            .withReaderGroupScope(getScope())
            .build();

    StreamExecutionEnvironment env = getStreamExecutionEnvironment();

    // create a TableEnvironment
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    tEnv.registerTableSource("TaxiRide", source);

    String fields = "vendorId, pickupTime, startLocationId, destLocationId, startLocationBorough, startLocationZone, destLocationBorough, destLocationZone";

    Table popularRides = tEnv
            .scan("TaxiRide")
            .select(fields)
            .window(Slide.over("15.minutes").every("5.minutes").on("pickupTime").as("w"))
            .groupBy("vendorId, w")
            .select("vendorId, w.start AS start, w.end AS end, count(vendorId) AS cnt");

    tEnv.toAppendStream(popularRides, Row.class).print();

    try {
        env.execute("Popular-Taxi-Vendor");
    } catch (Exception e) {
        log.error("Application Failed", e);
    }
}
 
Example 7
@Override
public void handleRequest() {

    TableSchema tableSchema = TripRecord.getTableSchema();

    FlinkPravegaJsonTableSource source = FlinkPravegaJsonTableSource.builder()
            .forStream(Stream.of(getScope(), getStream()).getScopedName())
            .withPravegaConfig(getPravegaConfig())
            .failOnMissingField(true)
            .withRowtimeAttribute("dropOffTime", new ExistingField("dropOffTime"), new BoundedOutOfOrderTimestamps(30000L))
            .withSchema(tableSchema)
            .withReaderGroupScope(getScope())
            .build();

    StreamExecutionEnvironment env = getStreamExecutionEnvironment();

    // create a TableEnvironment
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    tEnv.registerTableSource("TaxiRide", source);

    String fields = "passengerCount, dropOffTime, destLocationZone";

    Table noOfTravelersPerDest = tEnv
            .scan("TaxiRide")
            .select(fields)
            .window(Tumble.over("1.hour").on("dropOffTime").as("w"))
            .groupBy("destLocationZone, w")
            .select("destLocationZone, w.start AS start, w.end AS end, count(passengerCount) AS cnt");

    tEnv.toAppendStream(noOfTravelersPerDest, Row.class).print();

    try {
        env.execute("Max-Travellers-Per-Destination");
    } catch (Exception e) {
        log.error("Application Failed", e);
    }
}
 
Example 8
Source Project: flink   Source File: StreamSQLTestProgram.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<RowtimeAttributeDescriptor> getRowtimeAttributeDescriptors() {
	return Collections.singletonList(
		new RowtimeAttributeDescriptor(
			"rowtime",
			new ExistingField("ts"),
			new BoundedOutOfOrderTimestamps(100)));
}
 
Example 9
@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceUtil.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 
Example 10
Source Project: flink   Source File: KafkaTableSourceSinkFactoryTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testTableSource() {

	// prepare parameters for Kafka table source

	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, Types.STRING())
		.field(COUNT, Types.DECIMAL())
		.field(EVENT_TIME, Types.SQL_TIMESTAMP())
		.field(PROC_TIME, Types.SQL_TIMESTAMP())
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, Types.STRING())
			.field(COUNT, Types.DECIMAL())
			.field(TIME, Types.SQL_TIMESTAMP())
			.build()
			.toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets);

	TableSourceValidation.validateTableSource(expected);

	// construct table source using descriptors and table source factory

	final TestTableDescriptor testDesc = new TestTableDescriptor(
			new Kafka()
				.version(getKafkaVersion())
				.topic(TOPIC)
				.properties(KAFKA_PROPERTIES)
				.sinkPartitionerRoundRobin() // test if accepted although not needed
				.startFromSpecificOffsets(OFFSETS))
		.withFormat(new TestTableFormat())
		.withSchema(
			new Schema()
				.field(FRUIT_NAME, Types.STRING()).from(NAME)
				.field(COUNT, Types.DECIMAL()) // no from so it must match with the input
				.field(EVENT_TIME, Types.SQL_TIMESTAMP()).rowtime(
					new Rowtime().timestampsFromField(TIME).watermarksPeriodicAscending())
				.field(PROC_TIME, Types.SQL_TIMESTAMP()).proctime())
		.inAppendMode();

	final Map<String, String> propertiesMap = testDesc.toProperties();
	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}
 
Example 11
Source Project: pravega-samples   Source File: PopularDestinationQuery.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void handleRequest() {

    TableSchema tableSchema = TripRecord.getTableSchema();

    FlinkPravegaJsonTableSource source = FlinkPravegaJsonTableSource.builder()
            .forStream(Stream.of(getScope(), getStream()).getScopedName())
            .withPravegaConfig(getPravegaConfig())
            .failOnMissingField(true)
            .withRowtimeAttribute("pickupTime",
                    new ExistingField("pickupTime"),
                    new BoundedOutOfOrderTimestamps(30000L))
            .withSchema(tableSchema)
            .withReaderGroupScope(getScope())
            .build();


    StreamExecutionEnvironment env = getStreamExecutionEnvironment();

    // create a TableEnvironment
    StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
    tEnv.registerTableSource("TaxiRide", source);

    String query =
            "SELECT " +
                    "destLocationId, wstart, wend, cnt " +
                    "FROM " +
                    "(SELECT " +
                    "destLocationId, " +
                    "HOP_START(pickupTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wstart, " +
                    "HOP_END(pickupTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wend, " +
                    "COUNT(destLocationId) AS cnt " +
                    "FROM " +
                    "(SELECT " +
                    "pickupTime, " +
                    "destLocationId " +
                    "FROM TaxiRide) " +
                    "GROUP BY destLocationId, HOP(pickupTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE)) " +
                    "WHERE cnt > " + getLimit();

    Table results = tEnv.sqlQuery(query);

    tEnv.toAppendStream(results, Row.class).print();

    try {
        env.execute("Popular-Destination");
    } catch (Exception e) {
        log.error("Application Failed", e);
    }
}
 
Example 12
Source Project: flink   Source File: KafkaTableSourceSinkFactoryTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testTableSource() {
	// prepare parameters for Kafka table source
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(38, 18))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.field(PROC_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, DataTypes.STRING())
			.field(COUNT, DataTypes.DECIMAL(38, 18))
			.field(TIME, DataTypes.TIMESTAMP(3))
			.build().toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets,
		0L);

	TableSourceValidation.validateTableSource(expected, schema);

	// construct table source using descriptors and table source factory
	final Map<String, String> propertiesMap = new HashMap<>();
	propertiesMap.putAll(createKafkaSourceProperties());
	propertiesMap.put("schema.watermark.0.rowtime", EVENT_TIME);
	propertiesMap.put("schema.watermark.0.strategy.expr", WATERMARK_EXPRESSION);
	propertiesMap.put("schema.watermark.0.strategy.data-type", WATERMARK_DATATYPE.toString());
	propertiesMap.put("schema.4.name", COMPUTED_COLUMN_NAME);
	propertiesMap.put("schema.4.data-type", COMPUTED_COLUMN_DATATYPE.toString());
	propertiesMap.put("schema.4.expr", COMPUTED_COLUMN_EXPRESSION);

	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, propertiesMap)
		.createStreamTableSource(propertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
	// Test commitOnCheckpoints flag should be true when set consumer group.
	assertTrue(((FlinkKafkaConsumerBase) mock.sourceFunction).getEnableCommitOnCheckpoints());
}
 
Example 13
Source Project: flink   Source File: KafkaTableSourceSinkFactoryTestBase.java    License: Apache License 2.0 4 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testTableSourceWithLegacyProperties() {
	// prepare parameters for Kafka table source
	final TableSchema schema = TableSchema.builder()
		.field(FRUIT_NAME, DataTypes.STRING())
		.field(COUNT, DataTypes.DECIMAL(38, 18))
		.field(EVENT_TIME, DataTypes.TIMESTAMP(3))
		.field(PROC_TIME, DataTypes.TIMESTAMP(3))
		.build();

	final List<RowtimeAttributeDescriptor> rowtimeAttributeDescriptors = Collections.singletonList(
		new RowtimeAttributeDescriptor(EVENT_TIME, new ExistingField(TIME), new AscendingTimestamps()));

	final Map<String, String> fieldMapping = new HashMap<>();
	fieldMapping.put(FRUIT_NAME, NAME);
	fieldMapping.put(NAME, NAME);
	fieldMapping.put(COUNT, COUNT);
	fieldMapping.put(TIME, TIME);

	final Map<KafkaTopicPartition, Long> specificOffsets = new HashMap<>();
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_0), OFFSET_0);
	specificOffsets.put(new KafkaTopicPartition(TOPIC, PARTITION_1), OFFSET_1);

	final TestDeserializationSchema deserializationSchema = new TestDeserializationSchema(
		TableSchema.builder()
			.field(NAME, DataTypes.STRING())
			.field(COUNT, DataTypes.DECIMAL(38, 18))
			.field(TIME, DataTypes.TIMESTAMP(3))
			.build().toRowType()
	);

	final KafkaTableSourceBase expected = getExpectedKafkaTableSource(
		schema,
		Optional.of(PROC_TIME),
		rowtimeAttributeDescriptors,
		fieldMapping,
		TOPIC,
		KAFKA_PROPERTIES,
		deserializationSchema,
		StartupMode.SPECIFIC_OFFSETS,
		specificOffsets,
		0L);

	TableSourceValidation.validateTableSource(expected, schema);

	// construct table source using descriptors and table source factory
	final Map<String, String> legacyPropertiesMap = new HashMap<>();
	legacyPropertiesMap.putAll(createKafkaSourceProperties());

	// use legacy properties
	legacyPropertiesMap.remove("connector.specific-offsets");
	legacyPropertiesMap.remove("connector.properties.bootstrap.servers");
	legacyPropertiesMap.remove("connector.properties.group.id");

	// keep compatible with a specified update-mode
	legacyPropertiesMap.put("update-mode", "append");

	// legacy properties for specific-offsets and properties
	legacyPropertiesMap.put("connector.specific-offsets.0.partition", "0");
	legacyPropertiesMap.put("connector.specific-offsets.0.offset", "100");
	legacyPropertiesMap.put("connector.specific-offsets.1.partition", "1");
	legacyPropertiesMap.put("connector.specific-offsets.1.offset", "123");
	legacyPropertiesMap.put("connector.properties.0.key", "bootstrap.servers");
	legacyPropertiesMap.put("connector.properties.0.value", "dummy");
	legacyPropertiesMap.put("connector.properties.1.key", "group.id");
	legacyPropertiesMap.put("connector.properties.1.value", "dummy");

	final TableSource<?> actualSource = TableFactoryService.find(StreamTableSourceFactory.class, legacyPropertiesMap)
		.createStreamTableSource(legacyPropertiesMap);

	assertEquals(expected, actualSource);

	// test Kafka consumer
	final KafkaTableSourceBase actualKafkaSource = (KafkaTableSourceBase) actualSource;
	final StreamExecutionEnvironmentMock mock = new StreamExecutionEnvironmentMock();
	actualKafkaSource.getDataStream(mock);
	assertTrue(getExpectedFlinkKafkaConsumer().isAssignableFrom(mock.sourceFunction.getClass()));
}