Java Code Examples for org.apache.flink.table.api.EnvironmentSettings#Builder

The following examples show how to use org.apache.flink.table.api.EnvironmentSettings#Builder . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ExecutionEntry.java From flink with Apache License 2.0

6 votes

public EnvironmentSettings getEnvironmentSettings() {
	final EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance();

	if (inStreamingMode()) {
		builder.inStreamingMode();
	} else if (inBatchMode()) {
		builder.inBatchMode();
	}

	final String planner = properties.getOptionalString(EXECUTION_PLANNER)
		.orElse(EXECUTION_PLANNER_VALUE_OLD);

	if (planner.equals(EXECUTION_PLANNER_VALUE_OLD)) {
		builder.useOldPlanner();
	} else if (planner.equals(EXECUTION_PLANNER_VALUE_BLINK)) {
		builder.useBlinkPlanner();
	}

	return builder.build();
}

Example 2

Source File: ExecutionEntry.java From flink with Apache License 2.0

6 votes

public EnvironmentSettings getEnvironmentSettings() {
	final EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance();

	if (inStreamingMode()) {
		builder.inStreamingMode();
	} else if (inBatchMode()) {
		builder.inBatchMode();
	}

	final String planner = properties.getOptionalString(EXECUTION_PLANNER)
		.orElse(EXECUTION_PLANNER_VALUE_BLINK);

	if (planner.equals(EXECUTION_PLANNER_VALUE_OLD)) {
		builder.useOldPlanner();
	} else if (planner.equals(EXECUTION_PLANNER_VALUE_BLINK)) {
		builder.useBlinkPlanner();
	}

	return builder.build();
}

Example 3

Source File: HBaseTestBase.java From flink with Apache License 2.0

5 votes

@Before
public void before() {
	EnvironmentSettings.Builder streamBuilder = EnvironmentSettings.newInstance().inStreamingMode();
	EnvironmentSettings.Builder batchBuilder = EnvironmentSettings.newInstance().inBatchMode();
	if (BLINK_PLANNER.equals(planner)) {
		this.streamSettings = streamBuilder.useBlinkPlanner().build();
		this.batchSettings = batchBuilder.useBlinkPlanner().build();
	} else if (OLD_PLANNER.equals(planner)) {
		this.streamSettings = streamBuilder.useOldPlanner().build();
		this.batchSettings = batchBuilder.useOldPlanner().build();
	} else {
		throw new IllegalArgumentException("Unsupported planner name " + planner);
	}
}

Example 4

Source File: HBaseTestBase.java From flink with Apache License 2.0

5 votes

@Before
public void before() {
	EnvironmentSettings.Builder streamBuilder = EnvironmentSettings.newInstance().inStreamingMode();
	EnvironmentSettings.Builder batchBuilder = EnvironmentSettings.newInstance().inBatchMode();
	if (PlannerType.BLINK_PLANNER.equals(planner())) {
		this.streamSettings = streamBuilder.useBlinkPlanner().build();
		this.batchSettings = batchBuilder.useBlinkPlanner().build();
	} else if (PlannerType.OLD_PLANNER.equals(planner())) {
		this.streamSettings = streamBuilder.useOldPlanner().build();
		this.batchSettings = batchBuilder.useOldPlanner().build();
	} else {
		throw new IllegalArgumentException("Unsupported planner name " + planner());
	}
}

Example 5

Source File: HiveCatalogITCase.java From flink with Apache License 2.0

5 votes

private TableEnvironment prepareTable(boolean isStreaming) {
	EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance().useBlinkPlanner();
	if (isStreaming) {
		builder = builder.inStreamingMode();
	} else {
		builder = builder.inBatchMode();
	}
	EnvironmentSettings settings = builder.build();
	TableEnvironment tableEnv = TableEnvironment.create(settings);
	tableEnv.getConfig().getConfiguration().setInteger(TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1);

	tableEnv.registerCatalog("myhive", hiveCatalog);
	tableEnv.useCatalog("myhive");

	String srcPath = this.getClass().getResource("/csv/test3.csv").getPath();

	tableEnv.executeSql("CREATE TABLE proctime_src (" +
			"price DECIMAL(10, 2)," +
			"currency STRING," +
			"ts6 TIMESTAMP(6)," +
			"ts AS CAST(ts6 AS TIMESTAMP(3))," +
			"WATERMARK FOR ts AS ts," +
			"l_proctime AS PROCTIME( )) " + // test " " in proctime()
			String.format("WITH (" +
					"'connector.type' = 'filesystem'," +
					"'connector.path' = 'file://%s'," +
					"'format.type' = 'csv')", srcPath));

	return tableEnv;
}

Example 6

Source File: HiveCatalogITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testTableWithPrimaryKey() {
	EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance().useBlinkPlanner();
	EnvironmentSettings settings = builder.build();
	TableEnvironment tableEnv = TableEnvironment.create(settings);
	tableEnv.getConfig().getConfiguration().setInteger(TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 1);

	tableEnv.registerCatalog("catalog1", hiveCatalog);
	tableEnv.useCatalog("catalog1");

	final String createTable = "CREATE TABLE pk_src (\n" +
			"  uuid varchar(40) not null,\n" +
			"  price DECIMAL(10, 2),\n" +
			"  currency STRING,\n" +
			"  ts6 TIMESTAMP(6),\n" +
			"  ts AS CAST(ts6 AS TIMESTAMP(3)),\n" +
			"  WATERMARK FOR ts AS ts,\n" +
			"  constraint ct1 PRIMARY KEY(uuid) NOT ENFORCED)\n" +
			"  WITH (\n" +
			"    'connector.type' = 'filesystem'," +
			"    'connector.path' = 'file://fakePath'," +
			"    'format.type' = 'csv')";

	tableEnv.executeSql(createTable);

	TableSchema tableSchema = tableEnv.getCatalog(tableEnv.getCurrentCatalog())
			.map(catalog -> {
				try {
					final ObjectPath tablePath = ObjectPath.fromString(catalog.getDefaultDatabase() + '.' + "pk_src");
					return catalog.getTable(tablePath).getSchema();
				} catch (TableNotExistException e) {
					return null;
				}
			}).orElse(null);
	assertNotNull(tableSchema);
	assertEquals(
			tableSchema.getPrimaryKey(),
			Optional.of(UniqueConstraint.primaryKey("ct1", Collections.singletonList("uuid"))));
	tableEnv.executeSql("DROP TABLE pk_src");
}

Example 7

Source File: StreamSQLTestProgram.java From flink with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		String outputPath = params.getRequired("outputPath");
		String planner = params.get("planner", "old");

		final EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance();
		builder.inStreamingMode();

		if (planner.equals("old")) {
			builder.useOldPlanner();
		} else if (planner.equals("blink")) {
			builder.useBlinkPlanner();
		}

		final EnvironmentSettings settings = builder.build();

		final StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
		sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(
			3,
			Time.of(10, TimeUnit.SECONDS)
		));
		sEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		sEnv.enableCheckpointing(4000);
		sEnv.getConfig().setAutoWatermarkInterval(1000);

		final StreamTableEnvironment tEnv = StreamTableEnvironment.create(sEnv, settings);

		tEnv.registerTableSource("table1", new GeneratorTableSource(10, 100, 60, 0));
		tEnv.registerTableSource("table2", new GeneratorTableSource(5, 0.2f, 60, 5));

		int overWindowSizeSeconds = 1;
		int tumbleWindowSizeSeconds = 10;

		String overQuery = String.format(
			"SELECT " +
			"  key, " +
			"  rowtime, " +
			"  COUNT(*) OVER (PARTITION BY key ORDER BY rowtime RANGE BETWEEN INTERVAL '%d' SECOND PRECEDING AND CURRENT ROW) AS cnt " +
			"FROM table1",
			overWindowSizeSeconds);

		String tumbleQuery = String.format(
			"SELECT " +
			"  key, " +
			"  CASE SUM(cnt) / COUNT(*) WHEN 101 THEN 1 ELSE 99 END AS correct, " +
			"  TUMBLE_START(rowtime, INTERVAL '%d' SECOND) AS wStart, " +
			"  TUMBLE_ROWTIME(rowtime, INTERVAL '%d' SECOND) AS rowtime " +
			"FROM (%s) " +
			"WHERE rowtime > TIMESTAMP '1970-01-01 00:00:01' " +
			"GROUP BY key, TUMBLE(rowtime, INTERVAL '%d' SECOND)",
			tumbleWindowSizeSeconds,
			tumbleWindowSizeSeconds,
			overQuery,
			tumbleWindowSizeSeconds);

		String joinQuery = String.format(
			"SELECT " +
			"  t1.key, " +
			"  t2.rowtime AS rowtime, " +
			"  t2.correct," +
			"  t2.wStart " +
			"FROM table2 t1, (%s) t2 " +
			"WHERE " +
			"  t1.key = t2.key AND " +
			"  t1.rowtime BETWEEN t2.rowtime AND t2.rowtime + INTERVAL '%d' SECOND",
			tumbleQuery,
			tumbleWindowSizeSeconds);

		String finalAgg = String.format(
			"SELECT " +
			"  SUM(correct) AS correct, " +
			"  TUMBLE_START(rowtime, INTERVAL '20' SECOND) AS rowtime " +
			"FROM (%s) " +
			"GROUP BY TUMBLE(rowtime, INTERVAL '20' SECOND)",
			joinQuery);

		// get Table for SQL query
		Table result = tEnv.sqlQuery(finalAgg);
		// convert Table into append-only DataStream
		DataStream<Row> resultStream =
			tEnv.toAppendStream(result, Types.ROW(Types.INT, Types.SQL_TIMESTAMP));

		final StreamingFileSink<Row> sink = StreamingFileSink
			.forRowFormat(new Path(outputPath), (Encoder<Row>) (element, stream) -> {
				PrintStream out = new PrintStream(stream);
				out.println(element.toString());
			})
			.withBucketAssigner(new KeyBucketAssigner())
			.withRollingPolicy(OnCheckpointRollingPolicy.build())
			.build();

		resultStream
			// inject a KillMapper that forwards all records but terminates the first execution attempt
			.map(new KillMapper()).setParallelism(1)
			// add sink function
			.addSink(sink).setParallelism(1);

		sEnv.execute();
	}

Example 8

Source File: StreamSQLTestProgram.java From flink with Apache License 2.0

4 votes

public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		String outputPath = params.getRequired("outputPath");
		String planner = params.get("planner", "blink");

		final EnvironmentSettings.Builder builder = EnvironmentSettings.newInstance();
		builder.inStreamingMode();

		if (planner.equals("old")) {
			builder.useOldPlanner();
		} else if (planner.equals("blink")) {
			builder.useBlinkPlanner();
		}

		final EnvironmentSettings settings = builder.build();

		final StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
		sEnv.setRestartStrategy(RestartStrategies.fixedDelayRestart(
			3,
			Time.of(10, TimeUnit.SECONDS)
		));
		sEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		sEnv.enableCheckpointing(4000);
		sEnv.getConfig().setAutoWatermarkInterval(1000);

		final StreamTableEnvironment tEnv = StreamTableEnvironment.create(sEnv, settings);

		((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table1", new GeneratorTableSource(10, 100, 60, 0));
		((TableEnvironmentInternal) tEnv).registerTableSourceInternal("table2", new GeneratorTableSource(5, 0.2f, 60, 5));

		int overWindowSizeSeconds = 1;
		int tumbleWindowSizeSeconds = 10;

		String overQuery = String.format(
			"SELECT " +
			"  key, " +
			"  rowtime, " +
			"  COUNT(*) OVER (PARTITION BY key ORDER BY rowtime RANGE BETWEEN INTERVAL '%d' SECOND PRECEDING AND CURRENT ROW) AS cnt " +
			"FROM table1",
			overWindowSizeSeconds);

		String tumbleQuery = String.format(
			"SELECT " +
			"  key, " +
			"  CASE SUM(cnt) / COUNT(*) WHEN 101 THEN 1 ELSE 99 END AS correct, " +
			"  TUMBLE_START(rowtime, INTERVAL '%d' SECOND) AS wStart, " +
			"  TUMBLE_ROWTIME(rowtime, INTERVAL '%d' SECOND) AS rowtime " +
			"FROM (%s) " +
			"WHERE rowtime > TIMESTAMP '1970-01-01 00:00:01' " +
			"GROUP BY key, TUMBLE(rowtime, INTERVAL '%d' SECOND)",
			tumbleWindowSizeSeconds,
			tumbleWindowSizeSeconds,
			overQuery,
			tumbleWindowSizeSeconds);

		String joinQuery = String.format(
			"SELECT " +
			"  t1.key, " +
			"  t2.rowtime AS rowtime, " +
			"  t2.correct," +
			"  t2.wStart " +
			"FROM table2 t1, (%s) t2 " +
			"WHERE " +
			"  t1.key = t2.key AND " +
			"  t1.rowtime BETWEEN t2.rowtime AND t2.rowtime + INTERVAL '%d' SECOND",
			tumbleQuery,
			tumbleWindowSizeSeconds);

		String finalAgg = String.format(
			"SELECT " +
			"  SUM(correct) AS correct, " +
			"  TUMBLE_START(rowtime, INTERVAL '20' SECOND) AS rowtime " +
			"FROM (%s) " +
			"GROUP BY TUMBLE(rowtime, INTERVAL '20' SECOND)",
			joinQuery);

		// get Table for SQL query
		Table result = tEnv.sqlQuery(finalAgg);
		// convert Table into append-only DataStream
		DataStream<Row> resultStream =
			tEnv.toAppendStream(result, Types.ROW(Types.INT, Types.SQL_TIMESTAMP));

		final StreamingFileSink<Row> sink = StreamingFileSink
			.forRowFormat(new Path(outputPath), (Encoder<Row>) (element, stream) -> {
				PrintStream out = new PrintStream(stream);
				out.println(element.toString());
			})
			.withBucketAssigner(new KeyBucketAssigner())
			.withRollingPolicy(OnCheckpointRollingPolicy.build())
			.build();

		resultStream
			// inject a KillMapper that forwards all records but terminates the first execution attempt
			.map(new KillMapper()).setParallelism(1)
			// add sink function
			.addSink(sink).setParallelism(1);

		sEnv.execute();
	}