org.apache.flink.table.catalog.exceptions.CatalogException Java Examples

@Override
public void useDatabase(SessionContext session, String databaseName) throws SqlExecutionException {
	final ExecutionContext<?> context = getOrCreateExecutionContext(session);
	final TableEnvironment tableEnv = context
		.createEnvironmentInstance()
		.getTableEnvironment();

	context.wrapClassLoader(() -> {
		// Rely on TableEnvironment/CatalogManager to validate input
		try {
			tableEnv.useDatabase(databaseName);
		} catch (CatalogException e) {
			throw new SqlExecutionException("Failed to switch to database " + databaseName, e);
		}
		session.setCurrentDatabase(databaseName);
		return null;
	});
}
 

@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
		throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partition);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);
	ensureFullPartitionSpec(tablePath, partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		if (!ignoreIfExists) {
			throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec);
		}
	}

	partitions.get(tablePath).put(partitionSpec, partition.copy());
}
 

@Override
public List<String> listFunctions(String databaseName) throws DatabaseNotExistException, CatalogException {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "databaseName cannot be null or empty");

	// client.getFunctions() returns empty list when the database doesn't exist
	// thus we need to explicitly check whether the database exists or not
	if (!databaseExists(databaseName)) {
		throw new DatabaseNotExistException(getName(), databaseName);
	}

	try {
		// hive-1.x requires the pattern not being null, so pass a pattern that matches any name
		return client.getFunctions(databaseName, ".*");
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list functions in database %s", databaseName), e);
	}
}
 

@Override
public TypeInfo visit(VarCharType varCharType) {
	// Flink's StringType is defined as VARCHAR(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish StringType and a VARCHAR(Integer.MAX_VALUE) instance
	// Thus always treat VARCHAR(Integer.MAX_VALUE) as StringType
	if (varCharType.getLength() == Integer.MAX_VALUE) {
		return TypeInfoFactory.stringTypeInfo;
	}
	if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
							"The maximum length is %d",
							varCharType.getLength(), HiveVarchar.MAX_VARCHAR_LENGTH));
	}
	return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");
	checkNotNull(partitionSpec, "CatalogPartitionSpec cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// partition spec can be partial
		List<String> partialVals = MetaStoreUtils.getPvals(hiveTable.getPartitionKeys(), partitionSpec.getPartitionSpec());
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), partialVals,
			(short) -1).stream().map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// pass -1 as max_parts to fetch all partitions
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), (short) -1).stream()
			.map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 

@Override
public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
		throws TableNotExistException {
	checkNotNull(tablePath);
	checkNotNull(newTable);

	CatalogBaseTable existingTable = tables.get(tablePath);

	if (existingTable != null) {
		if (existingTable.getClass() != newTable.getClass()) {
			throw new CatalogException(
				String.format("Table types don't match. Existing table is '%s' and new table is '%s'.",
					existingTable.getClass().getName(), newTable.getClass().getName()));
		}

		tables.put(tablePath, newTable.copy());
	} else if (!ignoreIfNotExists) {
		throw new TableNotExistException(getName(), tablePath);
	}
}
 

@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 

/**
 * Sets the current database name that will be used when resolving a table path.
 * The database has to exist in the current catalog.
 *
 * @param databaseName database name to set as current database name
 * @throws CatalogException thrown if the database doesn't exist in the current catalog
 * @see CatalogManager#resolveTable(String...)
 * @see CatalogManager#setCurrentCatalog(String)
 */
public void setCurrentDatabase(String databaseName) {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "The database name cannot be null or empty.");

	if (!catalogs.get(currentCatalogName).databaseExists(databaseName)) {
		throw new CatalogException(format(
			"A database with name [%s] does not exist in the catalog: [%s].",
			databaseName,
			currentCatalogName));
	}

	if (!currentDatabaseName.equals(databaseName)) {
		currentDatabaseName = databaseName;

		LOG.info(
			"Set the current default database as [{}] in the current default catalog [{}].",
			currentCatalogName,
			currentDatabaseName);
	}
}
 

private static HiveConf createHiveConf(@Nullable String hiveConfDir) {
	LOG.info("Setting hive conf dir as {}", hiveConfDir);

	try {
		HiveConf.setHiveSiteLocation(
			hiveConfDir == null ?
				null : Paths.get(hiveConfDir, "hive-site.xml").toUri().toURL());
	} catch (MalformedURLException e) {
		throw new CatalogException(
			String.format("Failed to get hive-site.xml from %s", hiveConfDir), e);
	}

	// create HiveConf from hadoop configuration
	return new HiveConf(HadoopUtils.getHadoopConfiguration(new org.apache.flink.configuration.Configuration()),
		HiveConf.class);
}
 

@Override
public Set<String> getTableNames() {
	try {
		return new HashSet<>(catalog.listTables(databaseName));
	} catch (DatabaseNotExistException e) {
		throw new CatalogException(e);
	}
}
 

@Override
public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws PartitionNotExistException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (!partitionExists(tablePath, partitionSpec)) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}

	return partitions.get(tablePath).get(partitionSpec).copy();
}
 

@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);

	return new ArrayList<>(partitions.get(tablePath).keySet());
}
 

@Override
public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	return partitions.containsKey(tablePath) && partitions.get(tablePath).containsKey(partitionSpec);
}
 

@Override
public TypeInfo visit(CharType charType) {
	if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support char type with length of '%d'. " +
							"The maximum length is %d",
							charType.getLength(), HiveChar.MAX_CHAR_LENGTH));
	}
	return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
 

@Test
public void testRegisterCatalogWithExistingName() throws Exception {
	thrown.expect(CatalogException.class);

	CatalogManager manager = root()
		.builtin(
			database(BUILTIN_DEFAULT_DATABASE_NAME))
		.catalog(TEST_CATALOG_NAME, database(TEST_CATALOG_DEFAULT_DB_NAME))
		.build();

	manager.registerCatalog(TEST_CATALOG_NAME, new GenericInMemoryCatalog(TEST_CATALOG_NAME));
}
 

@Test
public void testRegisterExternalCatalogWithExistingName() throws Exception {
	thrown.expect(CatalogException.class);
	thrown.expectMessage("An external catalog named [test] already exists.");

	CatalogManager manager = root()
		.builtin(
			database(BUILTIN_DEFAULT_DATABASE_NAME))
		.catalog(TEST_CATALOG_NAME, database(TEST_CATALOG_DEFAULT_DB_NAME))
		.build();

	manager.registerExternalCatalog(TEST_CATALOG_NAME, CommonTestData.getInMemoryTestCatalog(false));
}
 

@Test
public void testCannotSetExternalCatalogAsDefault() throws Exception {
	thrown.expect(CatalogException.class);
	thrown.expectMessage("An external catalog cannot be set as the default one.");

	CatalogManager manager = root()
		.externalCatalog("ext")
		.build();
	manager.setCurrentCatalog("ext");
}
 

@Test
public void testSetNonExistingCurrentCatalog() throws Exception {
	thrown.expect(CatalogException.class);
	thrown.expectMessage("A catalog with name [nonexistent] does not exist.");

	CatalogManager manager = root().build();
	manager.setCurrentCatalog("nonexistent");
}
 

@Test
public void testSetNonExistingCurrentDatabase() throws Exception {
	thrown.expect(CatalogException.class);
	thrown.expectMessage("A database with name [nonexistent] does not exist in the catalog: [builtin].");

	CatalogManager manager = root().build();
	// This catalog does not exist in the builtin catalog
	manager.setCurrentDatabase("nonexistent");
}
 

@Test
public void testAlterTable_differentTypedTable() throws Exception {
	catalog.createDatabase(db1, createDb(), false);

	CatalogTable table = createTable();
	catalog.createTable(path1, table, false);

	exception.expect(CatalogException.class);
	exception.expectMessage(
		String.format("Table types don't match. " +
				"Existing table is '%s' and " +
				"new table is 'org.apache.flink.table.catalog.CatalogTest$TestTable'.",
			table.getClass().getName()));
	catalog.alterTable(path1, new TestTable(), false);
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	// Test HiveCatalogFactory.createCatalog
	// But not use it for testing purpose
	assertTrue(super.createCatalog(name, properties) != null);

	// Developers may already have their own production/testing hive-site.xml set in their environment,
	// and Flink tests should avoid using those hive-site.xml.
	// Thus, explicitly create a testing HiveConf for unit tests here
	Catalog hiveCatalog = HiveTestUtils.createHiveCatalog(name, properties.get(HiveCatalogValidator.CATALOG_HIVE_VERSION));

	// Creates an additional database to test tableEnv.useDatabase() will switch current database of the catalog
	hiveCatalog.open();
	try {
		hiveCatalog.createDatabase(
			ADDITIONAL_TEST_DATABASE,
			new CatalogDatabaseImpl(new HashMap<>(), null),
			false);
		hiveCatalog.createTable(
			new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
			new CatalogTableImpl(
				TableSchema.builder()
					.field("testcol", DataTypes.INT())
					.build(),
				new HashMap<String, String>() {{
					put(CatalogConfig.IS_GENERIC, String.valueOf(true));
				}},
				""
			),
			false
		);
	} catch (DatabaseAlreadyExistException | TableAlreadyExistException | DatabaseNotExistException e) {
		throw new CatalogException(e);
	}

	return hiveCatalog;
}
 

@Override
public Set<String> getTableNames() {
	try {
		return new HashSet<>(catalog.listTables(databaseName));
	} catch (DatabaseNotExistException e) {
		throw new CatalogException(e);
	}
}
 

private static void ensureTableAndPartitionMatch(Table hiveTable, CatalogPartition catalogPartition) {
	boolean tableIsGeneric = Boolean.valueOf(hiveTable.getParameters().get(CatalogConfig.IS_GENERIC));
	boolean partitionIsGeneric = Boolean.valueOf(catalogPartition.getProperties().get(CatalogConfig.IS_GENERIC));

	if (tableIsGeneric != partitionIsGeneric) {
		throw new CatalogException(String.format("Cannot handle %s partition for %s table",
			catalogPartition.getClass().getName(), tableIsGeneric ? "generic" : "non-generic"));
	}
}
 

@Override
public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
		throws PartitionNotExistException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		partitions.get(tablePath).remove(partitionSpec);
		partitionStats.get(tablePath).remove(partitionSpec);
		partitionColumnStats.get(tablePath).remove(partitionSpec);
	} else if (!ignoreIfNotExists) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}
}
 

@Override
public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
		throws TableNotExistException, TableAlreadyExistException, CatalogException {
	checkNotNull(tablePath, "tablePath cannot be null");
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(newTableName), "newTableName cannot be null or empty");

	try {
		// alter_table() doesn't throw a clear exception when target table doesn't exist.
		// Thus, check the table existence explicitly
		if (tableExists(tablePath)) {
			ObjectPath newPath = new ObjectPath(tablePath.getDatabaseName(), newTableName);
			// alter_table() doesn't throw a clear exception when new table already exists.
			// Thus, check the table existence explicitly
			if (tableExists(newPath)) {
				throw new TableAlreadyExistException(getName(), newPath);
			} else {
				Table table = getHiveTable(tablePath);
				table.setTableName(newTableName);
				client.alter_table(tablePath.getDatabaseName(), tablePath.getObjectName(), table);
			}
		} else if (!ignoreIfNotExists) {
			throw new TableNotExistException(getName(), tablePath);
		}
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to rename table %s", tablePath.getFullName()), e);
	}
}
 

@Override
public SimpleGenericUDAFParameterInfo createUDAFParameterInfo(ObjectInspector[] params, boolean isWindowing, boolean distinct, boolean allColumns) {
	try {
		Constructor constructor = SimpleGenericUDAFParameterInfo.class.getConstructor(ObjectInspector[].class,
				boolean.class, boolean.class, boolean.class);
		return (SimpleGenericUDAFParameterInfo) constructor.newInstance(params, isWindowing, distinct, allColumns);
	} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
		throw new CatalogException("Failed to create SimpleGenericUDAFParameterInfo", e);
	}
}
 

@Override
public List<String> listDatabases() throws CatalogException {
	try {
		return client.getAllDatabases();
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list all databases in %s", getName()), e);
	}
}
 

@Override
public void close() throws CatalogException {
	if (client != null) {
		client.close();
		client = null;
		LOG.info("Close connection to Hive metastore");
	}
}
 

@Override
public void open() throws CatalogException {
	if (client == null) {
		client = HiveMetastoreClientFactory.create(hiveConf, hiveVersion);
		LOG.info("Connected to Hive metastore");
	}

	if (!databaseExists(getDefaultDatabase())) {
		throw new CatalogException(String.format("Configured default database %s doesn't exist in catalog %s.",
			getDefaultDatabase(), getName()));
	}
}