org.apache.flink.table.catalog.Catalog Java Examples

The following examples show how to use org.apache.flink.table.catalog.Catalog. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FunctionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlterFunction() throws Exception {
	String create = "create function f3 as 'org.apache.flink.function.TestFunction'";
	String alter = "alter function f3 as 'org.apache.flink.function.TestFunction2'";

	ObjectPath objectPath = new ObjectPath("default_database", "f3");
	assertTrue(tEnv().getCatalog("default_catalog").isPresent());
	Catalog catalog = tEnv().getCatalog("default_catalog").get();
	tEnv().executeSql(create);
	CatalogFunction beforeUpdate = catalog.getFunction(objectPath);
	assertEquals("org.apache.flink.function.TestFunction", beforeUpdate.getClassName());

	tEnv().executeSql(alter);
	CatalogFunction afterUpdate = catalog.getFunction(objectPath);
	assertEquals("org.apache.flink.function.TestFunction2", afterUpdate.getClassName());
}
 
Example #2
Source File: CatalogITest.java    From pulsar-flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCatalogs() throws Exception {
    String inmemoryCatalog = "inmemorycatalog";
    String pulsarCatalog1 = "pulsarcatalog1";
    String pulsarCatalog2 = "pulsarcatalog2";

    ExecutionContext context = createExecutionContext(CATALOGS_ENVIRONMENT_FILE, getStreamingConfs());
    TableEnvironment tableEnv = context.createEnvironmentInstance().getTableEnvironment();

    assertEquals(tableEnv.getCurrentCatalog(), inmemoryCatalog);
    assertEquals(tableEnv.getCurrentDatabase(), "mydatabase");

    Catalog catalog = tableEnv.getCatalog(pulsarCatalog1).orElse(null);
    assertNotNull(catalog);
    assertTrue(catalog instanceof PulsarCatalog);
    tableEnv.useCatalog(pulsarCatalog1);
    assertEquals(tableEnv.getCurrentDatabase(), "public/default");

    catalog = tableEnv.getCatalog(pulsarCatalog2).orElse(null);
    assertNotNull(catalog);
    assertTrue(catalog instanceof PulsarCatalog);
    tableEnv.useCatalog(pulsarCatalog2);
    assertEquals(tableEnv.getCurrentDatabase(), "tn/ns");
}
 
Example #3
Source File: CatalogStatisticsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetStatsFromCatalog() throws Exception {
	EnvironmentSettings settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build();
	TableEnvironment tEnv = TableEnvironment.create(settings);
	tEnv.registerTableSource("T1", new TestTableSource(true, tableSchema));
	tEnv.registerTableSource("T2", new TestTableSource(true, tableSchema));
	Catalog catalog = tEnv.getCatalog(tEnv.getCurrentCatalog()).orElse(null);
	assertNotNull(catalog);

	catalog.alterTableStatistics(ObjectPath.fromString("default_database.T1"),
			new CatalogTableStatistics(100, 10, 1000L, 2000L), true);
	catalog.alterTableStatistics(ObjectPath.fromString("default_database.T2"),
			new CatalogTableStatistics(100000000, 1000, 1000000000L, 2000000000L), true);
	catalog.alterTableColumnStatistics(ObjectPath.fromString("default_database.T1"), createColumnStats(), true);
	catalog.alterTableColumnStatistics(ObjectPath.fromString("default_database.T2"), createColumnStats(), true);

	Table table = tEnv.sqlQuery("select * from T1, T2 where T1.s3 = T2.s3");
	String result = tEnv.explain(table);
	// T1 is broadcast side
	String expected = TableTestUtil.readFromResource("/explain/testGetStatsFromCatalog.out");
	assertEquals(expected, TableTestUtil.replaceStageId(result));
}
 
Example #4
Source File: FunctionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlterFunction() throws Exception {
	TableEnvironment tableEnv = getTableEnvironment();
	String create = "create function f3 as 'org.apache.flink.function.TestFunction'";
	String alter = "alter function f3 as 'org.apache.flink.function.TestFunction2'";

	ObjectPath objectPath = new ObjectPath("default_database", "f3");
	assertTrue(tableEnv.getCatalog("default_catalog").isPresent());
	Catalog catalog = tableEnv.getCatalog("default_catalog").get();
	tableEnv.sqlUpdate(create);
	CatalogFunction beforeUpdate = catalog.getFunction(objectPath);
	assertEquals("org.apache.flink.function.TestFunction", beforeUpdate.getClassName());

	tableEnv.sqlUpdate(alter);
	CatalogFunction afterUpdate = catalog.getFunction(objectPath);
	assertEquals("org.apache.flink.function.TestFunction2", afterUpdate.getClassName());
}
 
Example #5
Source File: TableFactoryUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link TableSource} from a {@link CatalogTable}.
 *
 * <p>It considers {@link Catalog#getFactory()} if provided.
 */
@SuppressWarnings("unchecked")
public static <T> TableSource<T> findAndCreateTableSource(
		Catalog catalog,
		ObjectIdentifier objectIdentifier,
		CatalogTable catalogTable,
		ReadableConfig configuration) {
	TableSourceFactory.Context context = new TableSourceFactoryContextImpl(
		objectIdentifier,
		catalogTable,
		configuration);
	Optional<TableFactory> factoryOptional = catalog.getTableFactory();
	if (factoryOptional.isPresent()) {
		TableFactory factory = factoryOptional.get();
		if (factory instanceof TableSourceFactory) {
			return ((TableSourceFactory<T>) factory).createTableSource(context);
		} else {
			throw new ValidationException("Cannot query a sink-only table. "
				+ "TableFactory provided by catalog must implement TableSourceFactory");
		}
	} else {
		return findAndCreateTableSource(context);
	}
}
 
Example #6
Source File: TableFactoryUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link TableSink} from a {@link CatalogTable}.
 *
 * <p>It considers {@link Catalog#getFactory()} if provided.
 */
@SuppressWarnings("unchecked")
public static <T> TableSink<T> findAndCreateTableSink(
		@Nullable Catalog catalog,
		ObjectIdentifier objectIdentifier,
		CatalogTable catalogTable,
		ReadableConfig configuration,
		boolean isStreamingMode) {
	TableSinkFactory.Context context = new TableSinkFactoryContextImpl(
		objectIdentifier,
		catalogTable,
		configuration,
		!isStreamingMode);
	if (catalog == null) {
		return findAndCreateTableSink(context);
	} else {
		return createTableSinkForCatalogTable(catalog, context)
			.orElseGet(() -> findAndCreateTableSink(context));
	}
}
 
Example #7
Source File: DatabaseCalciteSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
private FlinkStatistic getStatistic(
		boolean isTemporary,
		CatalogBaseTable catalogBaseTable,
		ObjectIdentifier tableIdentifier) {
	if (isTemporary || catalogBaseTable instanceof QueryOperationCatalogView) {
		return FlinkStatistic.UNKNOWN();
	}
	if (catalogBaseTable instanceof CatalogTable) {
		Catalog catalog = catalogManager.getCatalog(catalogName).get();
		return FlinkStatistic.builder()
			.tableStats(extractTableStats(catalog, tableIdentifier))
			// this is a temporary solution, FLINK-15123 will resolve this
			.uniqueKeys(extractUniqueKeys(catalogBaseTable.getSchema()))
			.build();
	} else {
		return FlinkStatistic.UNKNOWN();
	}
}
 
Example #8
Source File: DatabaseCalciteSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
private static TableStats extractTableStats(
		Catalog catalog,
		ObjectIdentifier objectIdentifier) {
	final ObjectPath tablePath = objectIdentifier.toObjectPath();
	try {
		CatalogTableStatistics tableStatistics = catalog.getTableStatistics(tablePath);
		CatalogColumnStatistics columnStatistics = catalog.getTableColumnStatistics(tablePath);
		return convertToTableStats(tableStatistics, columnStatistics);
	} catch (TableNotExistException e) {
		throw new ValidationException(format(
			"Could not get statistic for table: [%s, %s, %s]",
			objectIdentifier.getCatalogName(),
			tablePath.getDatabaseName(),
			tablePath.getObjectName()), e);
	}
}
 
Example #9
Source File: SqlToOperationConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlterTableAddPkConstraintEnforced() throws Exception {
	Catalog catalog = new GenericInMemoryCatalog("default", "default");
	catalogManager.registerCatalog("cat1", catalog);
	catalog.createDatabase("db1", new CatalogDatabaseImpl(new HashMap<>(), null), true);
	CatalogTable catalogTable = new CatalogTableImpl(
			TableSchema.builder()
					.field("a", DataTypes.STRING().notNull())
					.field("b", DataTypes.BIGINT().notNull())
					.field("c", DataTypes.BIGINT())
					.build(),
			new HashMap<>(),
			"tb1");
	catalogManager.setCurrentCatalog("cat1");
	catalogManager.setCurrentDatabase("db1");
	catalog.createTable(new ObjectPath("db1", "tb1"), catalogTable, true);
	// Test alter table add enforced
	thrown.expect(ValidationException.class);
	thrown.expectMessage("Flink doesn't support ENFORCED mode for PRIMARY KEY constaint. "
			+ "ENFORCED/NOT ENFORCED  controls if the constraint checks are performed on the "
			+ "incoming/outgoing data. Flink does not own the data therefore the "
			+ "only supported mode is the NOT ENFORCED mode");
	parse("alter table tb1 add constraint ct1 primary key(a, b)",
			SqlDialect.DEFAULT);
}
 
Example #10
Source File: SqlToOperationConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlterTableAddUniqueConstraint() throws Exception {
	Catalog catalog = new GenericInMemoryCatalog("default", "default");
	catalogManager.registerCatalog("cat1", catalog);
	catalog.createDatabase("db1", new CatalogDatabaseImpl(new HashMap<>(), null), true);
	CatalogTable catalogTable = new CatalogTableImpl(
			TableSchema.builder()
					.field("a", DataTypes.STRING().notNull())
					.field("b", DataTypes.BIGINT().notNull())
					.build(),
			new HashMap<>(),
			"tb1");
	catalogManager.setCurrentCatalog("cat1");
	catalogManager.setCurrentDatabase("db1");
	catalog.createTable(new ObjectPath("db1", "tb1"), catalogTable, true);
	// Test alter add table constraint.
	thrown.expect(UnsupportedOperationException.class);
	thrown.expectMessage("UNIQUE constraint is not supported yet");
	parse("alter table tb1 add constraint ct1 unique(a, b) not enforced",
			SqlDialect.DEFAULT);
}
 
Example #11
Source File: SqlToOperationConverterTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testAlterTableAddUniqueConstraintEnforced() throws Exception {
	Catalog catalog = new GenericInMemoryCatalog("default", "default");
	catalogManager.registerCatalog("cat1", catalog);
	catalog.createDatabase("db1", new CatalogDatabaseImpl(new HashMap<>(), null), true);
	CatalogTable catalogTable = new CatalogTableImpl(
			TableSchema.builder()
					.field("a", DataTypes.STRING().notNull())
					.field("b", DataTypes.BIGINT().notNull())
					.field("c", DataTypes.BIGINT())
					.build(),
			new HashMap<>(),
			"tb1");
	catalogManager.setCurrentCatalog("cat1");
	catalogManager.setCurrentDatabase("db1");
	catalog.createTable(new ObjectPath("db1", "tb1"), catalogTable, true);
	// Test alter table add enforced
	thrown.expect(UnsupportedOperationException.class);
	thrown.expectMessage("UNIQUE constraint is not supported yet");
	parse("alter table tb1 add constraint ct1 unique(a, b)",
			SqlDialect.DEFAULT);
}
 
Example #12
Source File: TableEnvironmentImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
private void replaceTableInternal(String name, CatalogBaseTable table) {
	try {
		ObjectPath path = new ObjectPath(catalogManager.getBuiltInDatabaseName(), name);
		Optional<Catalog> catalog = catalogManager.getCatalog(catalogManager.getBuiltInCatalogName());
		if (catalog.isPresent()) {
			catalog.get().alterTable(
				path,
				table,
				false);
		}
	} catch (Exception e) {
		throw new TableException("Could not register table", e);
	}
}
 
Example #13
Source File: TableEnvironmentTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConnect() throws Exception {
	final TableEnvironmentMock tableEnv = TableEnvironmentMock.getStreamingInstance();

	tableEnv
		.connect(new ConnectorDescriptorMock(TableSourceFactoryMock.CONNECTOR_TYPE_VALUE, 1, true))
		.withFormat(new FormatDescriptorMock("my_format", 1))
		.withSchema(new Schema()
			.field("my_field_0", "INT")
			.field("my_field_1", "BOOLEAN"))
		.inAppendMode()
		.registerTableSource("my_table");

	final Catalog catalog = tableEnv.getCatalog(EnvironmentSettings.DEFAULT_BUILTIN_CATALOG)
		.orElseThrow(AssertionError::new);

	final CatalogBaseTable table = catalog
		.getTable(new ObjectPath(EnvironmentSettings.DEFAULT_BUILTIN_DATABASE, "my_table"));

	assertThat(
		table.getSchema(),
		equalTo(
			TableSchema.builder()
				.field("my_field_0", DataTypes.INT())
				.field("my_field_1", DataTypes.BOOLEAN())
				.build()));

	final ConnectorCatalogTable<?, ?> connectorCatalogTable = (ConnectorCatalogTable<?, ?>) table;

	assertThat(
		connectorCatalogTable.getTableSource().isPresent(),
		equalTo(true));
}
 
Example #14
Source File: TableFactoryUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a table sink for a {@link CatalogTable} using table factory associated with the catalog.
 */
public static Optional<TableSink> createTableSinkForCatalogTable(Catalog catalog, CatalogTable catalogTable, ObjectPath tablePath) {
	TableFactory tableFactory = catalog.getTableFactory().orElse(null);
	if (tableFactory instanceof TableSinkFactory) {
		return Optional.ofNullable(((TableSinkFactory) tableFactory).createTableSink(tablePath, catalogTable));
	}
	return Optional.empty();
}
 
Example #15
Source File: CatalogStatisticsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void alterTableStatisticsWithUnknownRowCount(
		Catalog catalog,
		String tableName) throws TableNotExistException, TablePartitionedException {
	catalog.alterTableStatistics(new ObjectPath(databaseName, tableName),
			new CatalogTableStatistics(CatalogTableStatistics.UNKNOWN.getRowCount(), 1, 10000, 200000), true);
	catalog.alterTableColumnStatistics(new ObjectPath(databaseName, tableName), createColumnStats(), true);
}
 
Example #16
Source File: FlinkSqlParser.java    From sylph with Apache License 2.0 5 votes vote down vote up
private void translateJoin(JoinInfo joinInfo, Map<String, CreateTable> batchTables)
{
    Table streamTable = getTable(tableEnv, joinInfo.getStreamTable());
    RowTypeInfo streamRowType = (RowTypeInfo) streamTable.getSchema().toRowType();
    DataStream<Row> inputStream = tableEnv.toAppendStream(streamTable, org.apache.flink.types.Row.class);
    inputStream.getTransformation().setOutputType(streamRowType);

    //get batch table schema
    CreateTable batchTable = requireNonNull(batchTables.get(joinInfo.getBatchTable().getName()), "batch table [" + joinInfo.getJoinTableName() + "] not exits");
    RowTypeInfo batchTableRowType = StreamSqlUtil.schemaToRowTypeInfo(StreamSqlUtil.getTableSchema(batchTable));
    List<SelectField> joinSelectFields = getAllSelectFields(joinInfo, streamRowType, batchTableRowType);

    //It is recommended to do keyby first.
    JoinContext joinContext = JoinContextImpl.createContext(joinInfo, streamRowType, joinSelectFields);
    RealTimeTransForm transForm = getJoinTransForm(joinContext, batchTable);
    DataStream<Row> joinResultStream = AsyncFunctionHelper.translate(inputStream, transForm);

    //set schema
    RowTypeInfo rowTypeInfo = getJoinOutScheam(joinSelectFields);
    joinResultStream.getTransformation().setOutputType(rowTypeInfo);
    //--register tmp joinTable

    Catalog catalog = tableEnv.getCatalog(tableEnv.getCurrentCatalog()).get();
    if (catalog.tableExists(ObjectPath.fromString(joinInfo.getJoinTableName()))) {
        Table table = tableEnv.fromDataStream(joinResultStream);
        CatalogBaseTable tableTable = new QueryOperationCatalogView(table.getQueryOperation());
        try {
            catalog.createTable(ObjectPath.fromString(joinInfo.getJoinTableName()), tableTable, true);
        }
        catch (TableAlreadyExistException | DatabaseNotExistException e) {
            e.printStackTrace();
        }
        //tableEnv.replaceRegisteredTable(joinInfo.getJoinTableName(), new RelTable(table.getRelNode()));
    }
    else {
        tableEnv.registerDataStream(joinInfo.getJoinTableName(), joinResultStream);
    }
    //next update join select query
    joinQueryUpdate(joinInfo, rowTypeInfo.getFieldNames());
}
 
Example #17
Source File: CatalogStatisticsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void alterTableStatistics(
		Catalog catalog,
		String tableName) throws TableNotExistException, TablePartitionedException {
	catalog.alterTableStatistics(new ObjectPath(databaseName, tableName),
			new CatalogTableStatistics(100, 10, 1000L, 2000L), true);
	catalog.alterTableColumnStatistics(new ObjectPath(databaseName, tableName), createColumnStats(), true);
}
 
Example #18
Source File: JdbcCatalogFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	final DescriptorProperties prop = getValidatedProperties(properties);

	return new JdbcCatalog(
		name,
		prop.getString(CATALOG_DEFAULT_DATABASE),
		prop.getString(CATALOG_JDBC_USERNAME),
		prop.getString(CATALOG_JDBC_PASSWORD),
		prop.getString(CATALOG_JDBC_BASE_URL));
}
 
Example #19
Source File: JdbcCatalogFactoryTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void test() {
	final CatalogDescriptor catalogDescriptor =
		new JdbcCatalogDescriptor(PostgresCatalog.DEFAULT_DATABASE, TEST_USERNAME, TEST_PWD, baseUrl);

	final Map<String, String> properties = catalogDescriptor.toProperties();

	final Catalog actualCatalog = TableFactoryService.find(CatalogFactory.class, properties)
		.createCatalog(TEST_CATALOG_NAME, properties);

	checkEquals(catalog, (JdbcCatalog) actualCatalog);

	assertTrue(((JdbcCatalog) actualCatalog).getInternal() instanceof PostgresCatalog);
}
 
Example #20
Source File: HiveCatalogFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = getValidatedProperties(properties);

	final String defaultDatabase =
		descriptorProperties.getOptionalString(CATALOG_DEFAULT_DATABASE)
			.orElse(HiveCatalog.DEFAULT_DB);

	final Optional<String> hiveConfDir = descriptorProperties.getOptionalString(CATALOG_HIVE_CONF_DIR);

	final String version = descriptorProperties.getOptionalString(CATALOG_HIVE_VERSION).orElse(HiveShimLoader.getHiveVersion());

	return new HiveCatalog(name, defaultDatabase, hiveConfDir.orElse(null), version);
}
 
Example #21
Source File: SqlToOperationConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
/** Convert ALTER DATABASE statement. */
private Operation convertAlterDatabase(SqlAlterDatabase sqlAlterDatabase) {
	String[] fullDatabaseName = sqlAlterDatabase.fullDatabaseName();
	if (fullDatabaseName.length > 2) {
		throw new SqlConversionException("alter database identifier format error");
	}
	String catalogName = (fullDatabaseName.length == 1) ? catalogManager.getCurrentCatalog() : fullDatabaseName[0];
	String databaseName = (fullDatabaseName.length == 1) ? fullDatabaseName[0] : fullDatabaseName[1];
	final Map<String, String> properties;
	CatalogDatabase originCatalogDatabase;
	Optional<Catalog> catalog = catalogManager.getCatalog(catalogName);
	if (catalog.isPresent()) {
		try {
			originCatalogDatabase = catalog.get().getDatabase(databaseName);
			properties = new HashMap<>(originCatalogDatabase.getProperties());
		} catch (DatabaseNotExistException e) {
			throw new SqlConversionException(String.format("Database %s not exists", databaseName), e);
		}
	} else {
		throw new SqlConversionException(String.format("Catalog %s not exists", catalogName));
	}
	// set with properties
	sqlAlterDatabase.getPropertyList().getList().forEach(p ->
		properties.put(((SqlTableOption) p).getKeyString(), ((SqlTableOption) p).getValueString()));
	CatalogDatabase catalogDatabase = new CatalogDatabaseImpl(properties, originCatalogDatabase.getComment());
	return new AlterDatabaseOperation(catalogName, databaseName, catalogDatabase);
}
 
Example #22
Source File: SqlToOperationConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
/** Convert ALTER DATABASE statement. */
private Operation convertAlterDatabase(SqlAlterDatabase sqlAlterDatabase) {
	String[] fullDatabaseName = sqlAlterDatabase.fullDatabaseName();
	if (fullDatabaseName.length > 2) {
		throw new SqlConversionException("alter database identifier format error");
	}
	String catalogName = (fullDatabaseName.length == 1) ? catalogManager.getCurrentCatalog() : fullDatabaseName[0];
	String databaseName = (fullDatabaseName.length == 1) ? fullDatabaseName[0] : fullDatabaseName[1];
	Map<String, String> properties = new HashMap<>();
	CatalogDatabase originCatalogDatabase;
	Optional<Catalog> catalog = catalogManager.getCatalog(catalogName);
	if (catalog.isPresent()) {
		try {
			originCatalogDatabase = catalog.get().getDatabase(databaseName);
			properties.putAll(originCatalogDatabase.getProperties());
		} catch (DatabaseNotExistException e) {
			throw new SqlConversionException(String.format("Database %s not exists", databaseName), e);
		}
	} else {
		throw new SqlConversionException(String.format("Catalog %s not exists", catalogName));
	}
	// set with properties
	sqlAlterDatabase.getPropertyList().getList().forEach(p ->
		properties.put(((SqlTableOption) p).getKeyString(), ((SqlTableOption) p).getValueString()));
	CatalogDatabase catalogDatabase = new CatalogDatabaseImpl(properties, originCatalogDatabase.getComment());
	return new AlterDatabaseOperation(catalogName, databaseName, catalogDatabase);
}
 
Example #23
Source File: TableEnvironmentImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public String[] listTables() {
	String currentCatalogName = catalogManager.getCurrentCatalog();
	Optional<Catalog> currentCatalog = catalogManager.getCatalog(currentCatalogName);

	return currentCatalog.map(catalog -> {
		try {
			return catalog.listTables(catalogManager.getCurrentDatabase()).toArray(new String[0]);
		} catch (DatabaseNotExistException e) {
			throw new ValidationException("Current database does not exist", e);
		}
	}).orElseThrow(() ->
		new TableException(String.format("The current catalog %s does not exist.", currentCatalogName)));
}
 
Example #24
Source File: SqlToOperationConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
/** Convert CREATE CATALOG statement. */
private Operation convertCreateCatalog(SqlCreateCatalog sqlCreateCatalog) {
	String catalogName = sqlCreateCatalog.catalogName();

	// set with properties
	Map<String, String> properties = new HashMap<>();
	sqlCreateCatalog.getPropertyList().getList().forEach(p ->
		properties.put(((SqlTableOption) p).getKeyString(), ((SqlTableOption) p).getValueString()));

	final CatalogFactory factory =
		TableFactoryService.find(CatalogFactory.class, properties, this.getClass().getClassLoader());

	Catalog catalog = factory.createCatalog(catalogName, properties);
	return new CreateCatalogOperation(catalogName, catalog);
}
 
Example #25
Source File: FactoryUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link DynamicTableSource} from a {@link CatalogTable}.
 *
 * <p>It considers {@link Catalog#getFactory()} if provided.
 */
public static DynamicTableSource createTableSource(
		@Nullable Catalog catalog,
		ObjectIdentifier objectIdentifier,
		CatalogTable catalogTable,
		ReadableConfig configuration,
		ClassLoader classLoader) {
	final DefaultDynamicTableContext context = new DefaultDynamicTableContext(
		objectIdentifier,
		catalogTable,
		configuration,
		classLoader);
	try {
		final DynamicTableSourceFactory factory = getDynamicTableFactory(
			DynamicTableSourceFactory.class,
			catalog,
			context);
		return factory.createDynamicTableSource(context);
	} catch (Throwable t) {
		throw new ValidationException(
			String.format(
				"Unable to create a source for reading table '%s'.\n\n" +
				"Table options are:\n\n" +
				"%s",
				objectIdentifier.asSummaryString(),
				catalogTable.getOptions()
					.entrySet()
					.stream()
					.map(e -> stringifyOption(e.getKey(), e.getValue()))
					.sorted()
					.collect(Collectors.joining("\n"))),
			t);
	}
}
 
Example #26
Source File: FactoryUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link DynamicTableSink} from a {@link CatalogTable}.
 *
 * <p>It considers {@link Catalog#getFactory()} if provided.
 */
public static DynamicTableSink createTableSink(
		@Nullable Catalog catalog,
		ObjectIdentifier objectIdentifier,
		CatalogTable catalogTable,
		ReadableConfig configuration,
		ClassLoader classLoader) {
	final DefaultDynamicTableContext context = new DefaultDynamicTableContext(
		objectIdentifier,
		catalogTable,
		configuration,
		classLoader);
	try {
		final DynamicTableSinkFactory factory = getDynamicTableFactory(
			DynamicTableSinkFactory.class,
			catalog,
			context);
		return factory.createDynamicTableSink(context);
	} catch (Throwable t) {
		throw new ValidationException(
			String.format(
				"Unable to create a sink for writing table '%s'.\n\n" +
				"Table options are:\n\n" +
				"%s",
				objectIdentifier.asSummaryString(),
				catalogTable.getOptions()
					.entrySet()
					.stream()
					.map(e -> stringifyOption(e.getKey(), e.getValue()))
					.sorted()
					.collect(Collectors.joining("\n"))),
			t);
	}
}
 
Example #27
Source File: FactoryUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T extends DynamicTableFactory> T getDynamicTableFactory(
		Class<T> factoryClass,
		@Nullable Catalog catalog,
		DefaultDynamicTableContext context) {
	// catalog factory has highest precedence
	if (catalog != null) {
		final Factory factory = catalog.getFactory()
			.filter(f -> factoryClass.isAssignableFrom(f.getClass()))
			.orElse(null);
		if (factory != null) {
			return (T) factory;
		}
	}

	// fallback to factory discovery
	final String connectorOption = context.getCatalogTable()
		.getOptions()
		.get(CONNECTOR.key());
	if (connectorOption == null) {
		throw new ValidationException(
			String.format(
				"Table options do not contain an option key '%s' for discovering a connector.",
				CONNECTOR.key()));
	}
	try {
		return discoverFactory(context.getClassLoader(), factoryClass, connectorOption);
	} catch (ValidationException e) {
		throw new ValidationException(
			String.format(
				"Cannot discover a connector using option '%s'.",
				stringifyOption(CONNECTOR.key(), connectorOption)),
			e);
	}
}
 
Example #28
Source File: CatalogSchemaTable.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Create a CatalogSchemaTable instance.
 *
 * @param tableIdentifier Table identifier
 * @param lookupResult A result of catalog lookup
 * @param statistic Table statistics
 * @param catalog The catalog which the schema table belongs to
 * @param isStreaming If the table is for streaming mode
 */
public CatalogSchemaTable(
		ObjectIdentifier tableIdentifier,
		TableLookupResult lookupResult,
		FlinkStatistic statistic,
		Catalog catalog,
		boolean isStreaming) {
	this.tableIdentifier = tableIdentifier;
	this.lookupResult = lookupResult;
	this.statistic = statistic;
	this.catalog = catalog;
	this.isStreamingMode = isStreaming;
}
 
Example #29
Source File: TableFactoryUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a table sink for a {@link CatalogTable} using table factory associated with the catalog.
 */
public static Optional<TableSink> createTableSinkForCatalogTable(Catalog catalog, TableSinkFactory.Context context) {
	TableFactory tableFactory = catalog.getTableFactory().orElse(null);
	if (tableFactory instanceof TableSinkFactory) {
		return Optional.ofNullable(((TableSinkFactory) tableFactory).createTableSink(context));
	}
	return Optional.empty();
}
 
Example #30
Source File: TableEnvironmentImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void registerTableInternal(String name, CatalogBaseTable table) {
	try {
		checkValidTableName(name);
		ObjectPath path = new ObjectPath(catalogManager.getBuiltInDatabaseName(), name);
		Optional<Catalog> catalog = catalogManager.getCatalog(catalogManager.getBuiltInCatalogName());
		if (catalog.isPresent()) {
			catalog.get().createTable(
				path,
				table,
				false);
		}
	} catch (Exception e) {
		throw new TableException("Could not register table", e);
	}
}