Java Code Examples for org.apache.flink.api.java.io.jdbc.JDBCInputFormat

The following examples show how to use org.apache.flink.api.java.io.jdbc.JDBCInputFormat. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: marble   Source File: FlinkCollectionsEnvBenchMark.java    License: Apache License 2.0 6 votes vote down vote up
public double runSqlForSingleTable(int limit, String sql) throws Throwable {
  Stopwatch s = Stopwatch.createStarted();
  try (Connection connection = BenchMarkUtil.getDBConnection()) {
    String fetchSql = BenchMarkUtil.generateFetchSql("item1", "i_item_sk", limit);
    ResultSet resultSet = connection
        .createStatement()
        .executeQuery(fetchSql);
    RowTypeInfo rowTypeInfo = typeOfJdbc(resultSet.getMetaData());
    ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
    BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
    DataSet ds = env.createInput(
        JDBCInputFormat.buildJDBCInputFormat()
            .setDrivername(BenchMarkUtil.DB_DRIVER)
            .setDBUrl(BenchMarkUtil.DB_CONNECTION_URL)
            .setQuery(fetchSql)
            .setRowTypeInfo(rowTypeInfo)
            .finish()
    );
    ds.collect();
    tEnv.registerDataSet("item1", ds);
    s.stop();
    return s.elapsed(TimeUnit.MICROSECONDS) * 0.001 + sqlQuery(tEnv, sql);
  }
}
 
Example 2
Source Project: Alink   Source File: MySqlDB.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getStreamTable(String tableName, Params params, Long sessionId) throws Exception {
	if (!params.contains(MySqlSourceParams.SCHEMA_STR)) {
		return super.getStreamTable(tableName, params, sessionId);
	} else {
		TableSchema schema = CsvUtil.schemaStr2Schema(params.get(MySqlSourceParams.SCHEMA_STR));

		JDBCInputFormat inputFormat = JDBCInputFormat.buildJDBCInputFormat()
			.setUsername(getUserName())
			.setPassword(getPassword())
			.setDrivername(getDriverName())
			.setDBUrl(getDbUrl())
			.setQuery("select * from " + tableName)
			.setRowTypeInfo(new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames()))
			.finish();

		return DataStreamConversionUtil.toTable(
			sessionId,
			MLEnvironmentFactory.get(sessionId).getStreamExecutionEnvironment().createInput(inputFormat),
			schema.getFieldNames(), schema.getFieldTypes());
	}
}
 
Example 3
Source Project: Alink   Source File: MySqlDB.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getBatchTable(String tableName, Params params, Long sessionId) throws Exception {
	if (!params.contains(MySqlSourceParams.SCHEMA_STR)) {
		return super.getBatchTable(tableName, params, sessionId);
	} else {
		TableSchema schema = CsvUtil.schemaStr2Schema(params.get(MySqlSourceParams.SCHEMA_STR));

		JDBCInputFormat inputFormat = JDBCInputFormat.buildJDBCInputFormat()
			.setUsername(getUserName())
			.setPassword(getPassword())
			.setDrivername(getDriverName())
			.setDBUrl(getDbUrl())
			.setQuery("select * from " + tableName)
			.setRowTypeInfo(new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames()))
			.finish();

		return DataSetConversionUtil.toTable(sessionId,
			MLEnvironmentFactory.get(sessionId).getExecutionEnvironment().createInput(inputFormat),
			schema.getFieldNames(), schema.getFieldTypes());
	}
}
 
Example 4
Source Project: Alink   Source File: JdbcDB.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getStreamTable(String tableName, Params parameter, Long sessionId) throws Exception {
    TableSchema schema = getTableSchema(tableName);
    JDBCInputFormat inputFormat = JDBCInputFormat.buildJDBCInputFormat()
            .setUsername(getUserName())
            .setPassword(getPassword())
            .setDrivername(getDriverName())
            .setDBUrl(getDbUrl())
            .setQuery("select * from " + tableName)
            .setRowTypeInfo(new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames()))
            .finish();

    return DataStreamConversionUtil.toTable(
            sessionId,
            MLEnvironmentFactory.get(sessionId).getStreamExecutionEnvironment().createInput(inputFormat),
            schema.getFieldNames(), schema.getFieldTypes());
}
 
Example 5
Source Project: Alink   Source File: JdbcDB.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Table getBatchTable(String tableName, Params parameter, Long sessionId) throws Exception {
    TableSchema schema = getTableSchema(tableName);
    JDBCInputFormat inputFormat = JDBCInputFormat.buildJDBCInputFormat()
            .setUsername(getUserName())
            .setPassword(getPassword())
            .setDrivername(getDriverName())
            .setDBUrl(getDbUrl())
            .setQuery("select * from " + tableName)
            .setRowTypeInfo(new RowTypeInfo(schema.getFieldTypes(), schema.getFieldNames()))
            .finish();

    return DataSetConversionUtil.toTable(sessionId,
            MLEnvironmentFactory.get(sessionId).getExecutionEnvironment().createInput(inputFormat),
            schema.getFieldNames(), schema.getFieldTypes());
}
 
Example 6
Source Project: marble   Source File: FlinkCollectionsEnvBenchMark.java    License: Apache License 2.0 5 votes vote down vote up
public double runSqlForJoin(int limit, String sql) throws Throwable {
  Stopwatch s = Stopwatch.createStarted();
  try (Connection connection = BenchMarkUtil.getDBConnection()) {
    ExecutionEnvironment env = ExecutionEnvironment.createCollectionsEnvironment();
    BatchTableEnvironment tEnv = TableEnvironment.getTableEnvironment(env);
    String fetchSql1 = BenchMarkUtil.generateFetchSql("item1", "i_item_sk", limit);
    ResultSet resultSet1 = connection
        .createStatement()
        .executeQuery(fetchSql1);
    RowTypeInfo rowTypeInfo1 = typeOfJdbc(resultSet1.getMetaData());
    DataSet ds1 = env.createInput(
        JDBCInputFormat.buildJDBCInputFormat()
            .setDrivername(BenchMarkUtil.DB_DRIVER)
            .setDBUrl(BenchMarkUtil.DB_CONNECTION_URL)
            .setQuery(fetchSql1)
            .setRowTypeInfo(rowTypeInfo1)
            .finish()
    );
    ds1.collect();
    tEnv.registerDataSet("item1", ds1);

    String fetchSql2 = BenchMarkUtil.generateFetchSql("item2", "i_item_sk", limit);
    ResultSet resultSet2 = connection
        .createStatement()
        .executeQuery(fetchSql2);
    RowTypeInfo rowTypeInfo2 = typeOfJdbc(resultSet2.getMetaData());
    DataSet ds2 = env.createInput(
        JDBCInputFormat.buildJDBCInputFormat()
            .setDrivername(BenchMarkUtil.DB_DRIVER)
            .setDBUrl(BenchMarkUtil.DB_CONNECTION_URL)
            .setQuery(fetchSql2)
            .setRowTypeInfo(rowTypeInfo2)
            .finish()
    );
    ds2.collect();
    tEnv.registerDataSet("item2", ds2);
    s.stop();
    return s.elapsed(TimeUnit.MICROSECONDS) * 0.001 + sqlQuery(tEnv, sql);
  }
}