Java Code Examples for org.apache.flink.table.api.java.StreamTableEnvironment#registerFunction()

The following examples show how to use org.apache.flink.table.api.java.StreamTableEnvironment#registerFunction() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ExecutionContext.java From Flink-CEPplus with Apache License 2.0

6 votes

private void registerTemporalTable(TemporalTableEntry temporalTableEntry) {
	try {
		final Table table = tableEnv.scan(temporalTableEntry.getHistoryTable());
		final TableFunction<?> function = table.createTemporalTableFunction(
			temporalTableEntry.getTimeAttribute(),
			String.join(",", temporalTableEntry.getPrimaryKeyFields()));
		if (tableEnv instanceof StreamTableEnvironment) {
			StreamTableEnvironment streamTableEnvironment = (StreamTableEnvironment) tableEnv;
			streamTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
		} else {
			BatchTableEnvironment batchTableEnvironment = (BatchTableEnvironment) tableEnv;
			batchTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
		}
	} catch (Exception e) {
		throw new SqlExecutionException(
			"Invalid temporal table '" + temporalTableEntry.getName() + "' over table '" +
				temporalTableEntry.getHistoryTable() + ".\nCause: " + e.getMessage());
	}
}

Example 2

Source File: ExecutionContext.java From flink with Apache License 2.0

6 votes

private void registerTemporalTable(TemporalTableEntry temporalTableEntry) {
	try {
		final Table table = tableEnv.scan(temporalTableEntry.getHistoryTable());
		final TableFunction<?> function = table.createTemporalTableFunction(
			temporalTableEntry.getTimeAttribute(),
			String.join(",", temporalTableEntry.getPrimaryKeyFields()));
		if (tableEnv instanceof StreamTableEnvironment) {
			StreamTableEnvironment streamTableEnvironment = (StreamTableEnvironment) tableEnv;
			streamTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
		} else {
			BatchTableEnvironment batchTableEnvironment = (BatchTableEnvironment) tableEnv;
			batchTableEnvironment.registerFunction(temporalTableEntry.getName(), function);
		}
	} catch (Exception e) {
		throw new SqlExecutionException(
			"Invalid temporal table '" + temporalTableEntry.getName() + "' over table '" +
				temporalTableEntry.getHistoryTable() + ".\nCause: " + e.getMessage());
	}
}

Example 3

Source File: UDTFStreamOp.java From Alink with Apache License 2.0

6 votes

@Override
public UDTFStreamOp linkFrom(StreamOperator<?>... inputs) {
    if (null == getFunc() && null == getFuncName()) {
        throw new IllegalArgumentException("A TableFunction or a registered function name must be set using setFunc or setFuncName.");
    }
    StreamOperator<?> in = checkAndGetFirst(inputs);
    String[] reservedCols = ObjectUtils.defaultIfNull(getReservedCols(), in.getColNames());

    StreamTableEnvironment tEnv = MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamTableEnvironment();

    String funcName = getFuncName();
    if (null == funcName) {
        funcName = UDFHelper.generateRandomFuncName();
        tEnv.registerFunction(funcName, func);
    }

    String clause = UDFHelper.generateUDTFClause(in.getOutputTable().toString(), funcName,
        getOutputCols(), getSelectedCols(), reservedCols);
    this.setOutputTable(tEnv.sqlQuery(clause));
    return this;
}

Example 4

Source File: UDFStreamOp.java From Alink with Apache License 2.0

6 votes

@Override
public UDFStreamOp linkFrom(StreamOperator<?>... inputs) {
    if (null == getFunc() && null == getFuncName()) {
        throw new IllegalArgumentException("A ScalarFunction or a registered function name must be set using setFunc or setFuncName.");
    }
    StreamOperator<?> in = checkAndGetFirst(inputs);
    String[] reservedCols = ObjectUtils.defaultIfNull(getReservedCols(), in.getColNames());

    StreamTableEnvironment tEnv = MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamTableEnvironment();

    String funcName = getFuncName();
    if (null == funcName) {
        funcName = UDFHelper.generateRandomFuncName();
        tEnv.registerFunction(funcName, func);
    }

    String clause = UDFHelper.generateUDFClause(in.getOutputTable().toString(), funcName,
        getOutputCol(), getSelectedCols(), reservedCols);
    setOutputTable(tEnv.sqlQuery(clause));
    return this;
}

Example 5

Source File: HBaseConnectorITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testHBaseLookupFunction() throws Exception {
	StreamExecutionEnvironment streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
	StreamTableEnvironment streamTableEnv = StreamTableEnvironment.create(streamEnv, streamSettings);
	StreamITCase.clear();

	// prepare a source table
	DataStream<Row> ds = streamEnv.fromCollection(testData2).returns(testTypeInfo2);
	Table in = streamTableEnv.fromDataStream(ds, "a, b, c");
	streamTableEnv.registerTable("src", in);

	Map<String, String> tableProperties = hbaseTableProperties();
	TableSource source = TableFactoryService
		.find(HBaseTableFactory.class, tableProperties)
		.createTableSource(tableProperties);

	streamTableEnv.registerFunction("hbaseLookup", ((HBaseTableSource) source).getLookupFunction(new String[]{ROWKEY}));

	// perform a temporal table join query
	String sqlQuery = "SELECT a,family1.col1, family3.col3 FROM src, LATERAL TABLE(hbaseLookup(a))";
	Table result = streamTableEnv.sqlQuery(sqlQuery);

	DataStream<Row> resultSet = streamTableEnv.toAppendStream(result, Row.class);
	resultSet.addSink(new StreamITCase.StringSink<>());

	streamEnv.execute();

	List<String> expected = new ArrayList<>();
	expected.add("1,10,Welt-1");
	expected.add("2,20,Welt-2");
	expected.add("3,30,Welt-3");
	expected.add("3,30,Welt-3");

	StreamITCase.compareWithList(expected);
}

Example 6

Source File: AbstractFlinkClient.java From alchemy with Apache License 2.0

5 votes

private void register(StreamTableEnvironment env, String name, Object function) {
    if (function instanceof TableFunction) {
        env.registerFunction(name, (TableFunction) function);
    } else if (function instanceof AggregateFunction) {
        env.registerFunction(name, (AggregateFunction) function);
    } else if (function instanceof ScalarFunction) {
        env.registerFunction(name, (ScalarFunction) function);
    } else {
        throw new RuntimeException("Unknown UDF {} was found." + name);
    }
    LOGGER.info("register udf, name:{}, class:{}", name, function.getClass());
}

Example 7

Source File: JDBCLookupFunctionITCase.java From flink with Apache License 2.0

4 votes

@Test
public void test() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
	StreamITCase.clear();

	Table t = tEnv.fromDataStream(env.fromCollection(Arrays.asList(
				new Tuple2<>(1, 1),
				new Tuple2<>(1, 1),
				new Tuple2<>(2, 3),
				new Tuple2<>(2, 5),
				new Tuple2<>(3, 5),
				new Tuple2<>(3, 8)
			)), "id1, id2");

	tEnv.registerTable("T", t);

	JDBCTableSource.Builder builder = JDBCTableSource.builder()
			.setOptions(JDBCOptions.builder()
					.setDBUrl(DB_URL)
					.setTableName(LOOKUP_TABLE)
					.build())
			.setSchema(TableSchema.builder().fields(
					new String[]{"id1", "id2", "comment1", "comment2"},
					new DataType[]{DataTypes.INT(), DataTypes.INT(), DataTypes.STRING(), DataTypes.STRING()})
					.build());
	if (useCache) {
		builder.setLookupOptions(JDBCLookupOptions.builder()
				.setCacheMaxSize(1000).setCacheExpireMs(1000 * 1000).build());
	}
	tEnv.registerFunction("jdbcLookup",
			builder.build().getLookupFunction(t.getSchema().getFieldNames()));

	String sqlQuery = "SELECT id1, id2, comment1, comment2 FROM T, " +
			"LATERAL TABLE(jdbcLookup(id1, id2)) AS S(l_id1, l_id2, comment1, comment2)";
	Table result = tEnv.sqlQuery(sqlQuery);

	DataStream<Row> resultSet = tEnv.toAppendStream(result, Row.class);
	resultSet.addSink(new StreamITCase.StringSink<>());
	env.execute();

	List<String> expected = new ArrayList<>();
	expected.add("1,1,11-c1-v1,11-c2-v1");
	expected.add("1,1,11-c1-v1,11-c2-v1");
	expected.add("1,1,11-c1-v2,11-c2-v2");
	expected.add("1,1,11-c1-v2,11-c2-v2");
	expected.add("2,3,null,23-c2");
	expected.add("2,5,25-c1,25-c2");
	expected.add("3,8,38-c1,38-c2");

	StreamITCase.compareWithList(expected);
}

Example 8

Source File: AreasTotalPerHourOfDay.java From infoworld-post with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

        // check parameter
        if (args.length != 1) {
            System.err.println("Please provide the path to the taxi rides file as a parameter");
        }
        String inputPath = args[0];

        // create execution environment
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // configure event-time and watermarks
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(1000L);

        // create table environment
        StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
        // register user-defined function
        tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());

        // get taxi ride event stream
        DataStream<TaxiRide> rides = TaxiRides.getRides(env, inputPath);
        // register taxi ride event stream as table "Rides"
        tEnv.registerDataStream(
            "Rides",
            rides,
            "medallion, licenseId, pickUpTime, dropOffTime.rowtime, " +
                "pickUpLon, pickUpLat, dropOffLon, dropOffLat, total");

        // define SQL query to compute average total per area and hour of day.
        Table result = tEnv.sqlQuery(
            "SELECT " +
            "  toCellId(dropOffLon, dropOffLat) AS area, " +
            "  EXTRACT(HOUR FROM dropOffTime) AS hourOfDay, " +
            "  AVG(total) AS avgTotal " +
            "FROM Rides " +
            "GROUP BY " +
            "  toCellId(dropOffLon, dropOffLat), " +
            "  EXTRACT(HOUR FROM dropOffTime)"
        );

        // convert result table into a retraction stream and print it
        tEnv.toRetractStream(result, Row.class)
                .print();

        // execute the query
        env.execute();
    }

Example 9

Source File: AreasTotalPerHour.java From infoworld-post with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

        // check parameter
        if (args.length != 1) {
            System.err.println("Please provide the path to the taxi rides file as a parameter");
        }
        String inputPath = args[0];

        // create execution environment
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        // configure event-time and watermarks
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.getConfig().setAutoWatermarkInterval(1000L);

        // create table environment
        StreamTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
        // register user-defined function
        tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());

        // get taxi ride event stream
        DataStream<TaxiRide> rides = TaxiRides.getRides(env, inputPath);
        // register taxi ride event stream as table "Rides"
        tEnv.registerDataStream(
            "Rides",
            rides,
            "medallion, licenseId, pickUpTime, dropOffTime.rowtime, " +
                "pickUpLon, pickUpLat, dropOffLon, dropOffLat, total");

        // define SQL query to compute average total per area and hour
        Table result = tEnv.sqlQuery(
            "SELECT " +
            "  toCellId(dropOffLon, dropOffLat) AS area, " +
            "  TUMBLE_START(dropOffTime, INTERVAL '1' HOUR) AS t, " +
            "  AVG(total) AS avgTotal " +
            "FROM Rides " +
            "GROUP BY " +
            "  toCellId(dropOffLon, dropOffLat), " +
            "  TUMBLE(dropOffTime, INTERVAL '1' HOUR)"
        );

        // convert result table into an append stream and print it
        tEnv.toAppendStream(result, Row.class)
            .print();

        // execute the query
        env.execute();
    }

Example 10

Source File: TestTableFunction.java From yauaa with Apache License 2.0

4 votes

@Test
public void testFunctionExtractDirect() throws Exception {
    // The base input stream
    StreamExecutionEnvironment                       senv        = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Tuple3<String, String, String>> inputStream = getTestAgentStream(senv);

    // The table environment
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);

    // Give the stream a Table Name
    tableEnv.createTemporaryView("AgentStream", inputStream, "useragent, expectedDeviceClass, expectedAgentNameVersionMajor");

    // register the function
    tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));


    // The downside of doing it this way is that the parsing function (i.e. parsing and converting all results into a map)
    // is called for each field you want. So in this simple case twice.
    String sqlQuery =
        "SELECT useragent,"+
        "       ParseUserAgent(useragent)['DeviceClass'          ]  as DeviceClass," +
        "       ParseUserAgent(useragent)['AgentNameVersionMajor']  as AgentNameVersionMajor," +
        "       expectedDeviceClass," +
        "       expectedAgentNameVersionMajor " +
        "FROM AgentStream";
    Table  resultTable   = tableEnv.sqlQuery(sqlQuery);

    TypeInformation<Row> tupleType = new RowTypeInfo(STRING, STRING, STRING, STRING, STRING);
    DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);

    resultSet.map((MapFunction<Row, String>) row -> {
        Object useragent                      = row.getField(0);
        Object deviceClass                    = row.getField(1);
        Object agentNameVersionMajor          = row.getField(2);
        Object expectedDeviceClass            = row.getField(3);
        Object expectedAgentNameVersionMajor  = row.getField(4);

        assertTrue(useragent                     instanceof String);
        assertTrue(deviceClass                   instanceof String);
        assertTrue(agentNameVersionMajor         instanceof String);
        assertTrue(expectedDeviceClass           instanceof String);
        assertTrue(expectedAgentNameVersionMajor instanceof String);

        assertEquals(expectedDeviceClass,           deviceClass,           "Wrong DeviceClass: "           + useragent);
        assertEquals(expectedAgentNameVersionMajor, agentNameVersionMajor, "Wrong AgentNameVersionMajor: " + useragent);
        return useragent.toString();
    }).printToErr();

    senv.execute();
}

Example 11

Source File: TestTableFunction.java From yauaa with Apache License 2.0

4 votes

@Test
public void testMapFunctionExtractInSQLSubSelect() throws Exception {
    // The base input stream
    StreamExecutionEnvironment                       senv        = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Tuple3<String, String, String>> inputStream = getTestAgentStream(senv);

    // The table environment
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);

    // Give the stream a Table Name
    tableEnv.createTemporaryView("AgentStream", inputStream, "useragent, expectedDeviceClass, expectedAgentNameVersionMajor");

    // register the function
    tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));

    String sqlQuery =
        "SELECT useragent,"+
        "       parsedUseragent['DeviceClass']              AS deviceClass," +
        "       parsedUseragent['AgentNameVersionMajor']    AS agentNameVersionMajor," +
        "       expectedDeviceClass," +
        "       expectedAgentNameVersionMajor " +
        "FROM ( " +
        "   SELECT useragent," +
        "          ParseUserAgent(useragent) AS parsedUseragent," +
        "          expectedDeviceClass," +
        "          expectedAgentNameVersionMajor " +
        "   FROM   AgentStream " +
        ")";

    Table  resultTable   = tableEnv.sqlQuery(sqlQuery);

    TypeInformation<Row> tupleType = new RowTypeInfo(STRING, STRING, STRING, STRING, STRING);
    DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);

    resultSet.map((MapFunction<Row, String>) row -> {
        Object useragent                      = row.getField(0);
        Object deviceClass                    = row.getField(1);
        Object agentNameVersionMajor          = row.getField(2);
        Object expectedDeviceClass            = row.getField(3);
        Object expectedAgentNameVersionMajor  = row.getField(4);

        assertTrue(useragent                     instanceof String);
        assertTrue(deviceClass                   instanceof String);
        assertTrue(agentNameVersionMajor         instanceof String);
        assertTrue(expectedDeviceClass           instanceof String);
        assertTrue(expectedAgentNameVersionMajor instanceof String);

        assertEquals(expectedDeviceClass,           deviceClass,           "Wrong DeviceClass: "           + useragent);
        assertEquals(expectedAgentNameVersionMajor, agentNameVersionMajor, "Wrong AgentNameVersionMajor: " + useragent);
        return useragent.toString();
    }).printToErr();

    senv.execute();
}

Example 12

Source File: TestTableFunction.java From yauaa with Apache License 2.0

4 votes

@Test
public void testMapFunctionReturnMap() throws Exception {
    // The base input stream
    StreamExecutionEnvironment                       senv        = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStreamSource<Tuple3<String, String, String>> inputStream = getTestAgentStream(senv);

    // The table environment
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);

    // Give the stream a Table Name
    tableEnv.createTemporaryView("AgentStream", inputStream, "useragent, expectedDeviceClass, expectedAgentNameVersionMajor");

    // register the function
    tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));

    String sqlQuery =
        "SELECT useragent," +
        "       ParseUserAgent(useragent)        AS parsedUseragent," +
        "       expectedDeviceClass              AS expectedDeviceClass," +
        "       expectedAgentNameVersionMajor    AS expectedAgentNameVersionMajor " +
        "FROM   AgentStream";

    Table  resultTable   = tableEnv.sqlQuery(sqlQuery);

    TypeInformation<Row> tupleType = new RowTypeInfo(STRING, MAP(STRING, STRING), STRING, STRING);
    DataStream<Row> resultSet = tableEnv.toAppendStream(resultTable, tupleType);

    resultSet.map((MapFunction<Row, String>) row -> {
        Object useragent                     = row.getField(0);
        Object parsedUseragent               = row.getField(1);
        Object expectedDeviceClass           = row.getField(2);
        Object expectedAgentNameVersionMajor = row.getField(3);

        assertTrue(useragent                     instanceof String);
        assertTrue(parsedUseragent               instanceof Map<?, ?>);
        assertTrue(expectedDeviceClass           instanceof String);
        assertTrue(expectedAgentNameVersionMajor instanceof String);

        assertEquals(
            expectedDeviceClass,
            ((Map<?, ?>)parsedUseragent).get("DeviceClass"),
            "Wrong DeviceClass: "           + useragent);

        assertEquals(
            expectedAgentNameVersionMajor,
            ((Map<?, ?>)parsedUseragent).get("AgentNameVersionMajor"),
            "Wrong AgentNameVersionMajor: " + useragent);

        return useragent.toString();
    }).printToErr();

    senv.execute();
}

Example 13

Source File: DemonstrationOfTumblingTableSQLFunction.java From yauaa with Apache License 2.0

4 votes

@Disabled
@Test
public void runDemonstration() throws Exception {
    // The base input stream
    StreamExecutionEnvironment senv = StreamExecutionEnvironment.getExecutionEnvironment();
    senv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    senv.getConfig().setAutoWatermarkInterval(1000);

    DataStream<Tuple4<Long, String, String, String>> inputStream = senv
        .addSource(new UAStreamSource())
        .assignTimestampsAndWatermarks(new UAWatermarker());

    // The table environment
    StreamTableEnvironment tableEnv = StreamTableEnvironment.create(senv);

    // Give the stream a Table Name
    tableEnv.createTemporaryView("AgentStream", inputStream, "eventTime.rowtime, useragent, expectedDeviceClass, expectedAgentNameVersionMajor");

    // register the function
    tableEnv.registerFunction("ParseUserAgent", new AnalyzeUseragentFunction("DeviceClass", "AgentNameVersionMajor"));

    int windowIntervalCount =  5;
    String windowIntervalScale =  "MINUTE";

    String sqlQuery = String.format(
        "SELECT" +
        "   TUMBLE_START(eventTime, INTERVAL '%d' %s) AS wStart," +
        "   deviceClass," +
        "   agentNameVersionMajor," +
        "   expectedDeviceClass," +
        "   expectedAgentNameVersionMajor," +
        "   Count('') " +
        "FROM ( "+
        "    SELECT " +
        "       eventTime, " +
        "       parsedUserAgent['DeviceClass'          ]  AS deviceClass," +
        "       parsedUserAgent['AgentNameVersionMajor']  AS agentNameVersionMajor," +
        "       expectedDeviceClass," +
        "       expectedAgentNameVersionMajor" +
        "    FROM ( "+
        "        SELECT " +
        "           eventTime, " +
        "           ParseUserAgent(useragent) AS parsedUserAgent," +
        "           expectedDeviceClass," +
        "           expectedAgentNameVersionMajor" +
        "        FROM AgentStream " +
        "    )" +
        ")" +
        "GROUP BY TUMBLE(eventTime, INTERVAL '%d' %s), " +
            "       deviceClass," +
            "       agentNameVersionMajor," +
            "       expectedDeviceClass," +
            "       expectedAgentNameVersionMajor",
        windowIntervalCount, windowIntervalScale,
        windowIntervalCount, windowIntervalScale
        );
    Table resultTable = tableEnv.sqlQuery(sqlQuery);

    TypeInformation<Row> tupleType = new RowTypeInfo(SQL_TIMESTAMP, STRING, STRING, STRING, STRING, LONG);
    DataStream<Row>      resultSet = tableEnv.toAppendStream(resultTable, tupleType);

    resultSet.print();

    resultSet.map((MapFunction<Row, String>) row -> {
        Object useragent                      = row.getField(0);
        Object deviceClass                    = row.getField(1);
        Object agentNameVersionMajor          = row.getField(2);
        Object expectedDeviceClass            = row.getField(3);
        Object expectedAgentNameVersionMajor  = row.getField(4);

        assertEquals(
            expectedDeviceClass,
            deviceClass,
            "Wrong DeviceClass: " + useragent);

        assertEquals(
            expectedAgentNameVersionMajor,
            agentNameVersionMajor,
            "Wrong AgentNameVersionMajor: " + useragent);

        return useragent.toString();
    });

    senv.execute();
}

Example 14

Source File: PopularPlacesSql.java From flink-training-exercises with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		// read parameters
		ParameterTool params = ParameterTool.fromArgs(args);
		String input = params.getRequired("input");

		final int maxEventDelay = 60;       // events are out of order by max 60 seconds
		final int servingSpeedFactor = 600; // events of 10 minutes are served in 1 second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		// create a TableEnvironment
		StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

		// register TaxiRideTableSource as table "TaxiRides"
		tEnv.registerTableSource(
				"TaxiRides",
				new TaxiRideTableSource(
						input,
						maxEventDelay,
						servingSpeedFactor));

		// register user-defined functions
		tEnv.registerFunction("isInNYC", new GeoUtils.IsInNYC());
		tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
		tEnv.registerFunction("toCoords", new GeoUtils.ToCoords());

		Table results = tEnv.sqlQuery(
			"SELECT " +
				"toCoords(cell), wstart, wend, isStart, popCnt " +
			"FROM " +
				"(SELECT " +
					"cell, " +
					"isStart, " +
					"HOP_START(eventTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wstart, " +
					"HOP_END(eventTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE) AS wend, " +
					"COUNT(isStart) AS popCnt " +
				"FROM " +
					"(SELECT " +
						"eventTime, " +
						"isStart, " +
						"CASE WHEN isStart THEN toCellId(startLon, startLat) ELSE toCellId(endLon, endLat) END AS cell " +
					"FROM TaxiRides " +
					"WHERE isInNYC(startLon, startLat) AND isInNYC(endLon, endLat)) " +
				"GROUP BY cell, isStart, HOP(eventTime, INTERVAL '5' MINUTE, INTERVAL '15' MINUTE)) " +
			"WHERE popCnt > 20"
			);

		// convert Table into an append stream and print it
		// (if instead we needed a retraction stream we would use tEnv.toRetractStream)
		tEnv.toAppendStream(results, Row.class).print();

		// execute query
		env.execute();
	}

Example 15

Source File: PopularPlacesTableApi.java From flink-training-exercises with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		// read parameters
		ParameterTool params = ParameterTool.fromArgs(args);
		String input = params.getRequired("input");

		final int maxEventDelay = 60;       // events are out of order by max 60 seconds
		final int servingSpeedFactor = 600; // events of 10 minutes are served in 1 second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		// create a TableEnvironment
		StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

		// register TaxiRideTableSource as table "TaxiRides"
		tEnv.registerTableSource(
				"TaxiRides",
				new TaxiRideTableSource(
						input,
						maxEventDelay,
						servingSpeedFactor));

		// register user-defined functions
		tEnv.registerFunction("isInNYC", new GeoUtils.IsInNYC());
		tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
		tEnv.registerFunction("toCoords", new GeoUtils.ToCoords());

		Table popPlaces = tEnv
				// scan TaxiRides table
				.scan("TaxiRides")
				// filter for valid rides
				.filter("isInNYC(startLon, startLat) && isInNYC(endLon, endLat)")
				// select fields and compute grid cell of departure or arrival coordinates
				.select("eventTime, " +
						"isStart, " +
						"(isStart = true).?(toCellId(startLon, startLat), toCellId(endLon, endLat)) AS cell")
				// specify sliding window over 15 minutes with slide of 5 minutes
				.window(Slide.over("15.minutes").every("5.minutes").on("eventTime").as("w"))
				// group by cell, isStart, and window
				.groupBy("cell, isStart, w")
				// count departures and arrivals per cell (location) and window (time)
				.select("cell, isStart, w.start AS start, w.end AS end, count(isStart) AS popCnt")
				// filter for popular places
				.filter("popCnt > 20")
				// convert cell back to coordinates
				.select("toCoords(cell) AS location, start, end, isStart, popCnt");

		// convert Table into an append stream and print it
		tEnv.toAppendStream(popPlaces, Row.class).print();

		// execute query
		env.execute();
	}

Example 16

Source File: RidesPerHour.java From flink-training-exercises with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		// read parameters
		ParameterTool params = ParameterTool.fromArgs(args);
		String input = params.getRequired("input");

		final int maxEventDelay = 60;       	// events are out of order by max 60 seconds
		final int servingSpeedFactor = 1800; 	// events of 30 minutes are served in 1 second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

		// create a TableEnvironment
		StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);

		// register TaxiRideTableSource as table "TaxiRides"
		tEnv.registerTableSource(
				"TaxiRides",
				new TaxiRideTableSource(
						input,
						maxEventDelay,
						servingSpeedFactor));

		// register user-defined functions
		tEnv.registerFunction("isInNYC", new GeoUtils.IsInNYC());
		tEnv.registerFunction("toCellId", new GeoUtils.ToCellId());
		tEnv.registerFunction("toCoords", new GeoUtils.ToCoords());

		Table results = tEnv.sqlQuery(
				//"SELECT TUMBLE_START(eventTime, INTERVAL '1' HOUR), isStart, count(isStart) FROM TaxiRides GROUP BY isStart, TUMBLE(eventTime, INTERVAL '1' HOUR)"
				//"SELECT avg(endTime - startTime), passengerCnt FROM TaxiRides GROUP BY passengerCnt"
				"SELECT CAST (toCellId(endLon, endLat) AS VARCHAR), eventTime," +
				"COUNT(*) OVER (" +
					"PARTITION BY toCellId(endLon, endLat) ORDER BY eventTime RANGE BETWEEN INTERVAL '10' MINUTE PRECEDING AND CURRENT ROW" +
				") " +
				"FROM( SELECT * FROM TaxiRides WHERE not isStart AND toCellId(endLon, endLat) = 50801 )"
		);

		// convert Table into an append stream and print it
		// (if instead we needed a retraction stream we would use tEnv.toRetractStream)
		tEnv.toRetractStream(results, Row.class).print();

		// execute query
		env.execute();
	}