Java Code Examples for org.apache.flink.table.catalog.exceptions.CatalogException

The following examples show how to use org.apache.flink.table.catalog.exceptions.CatalogException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: flink   Author: flink-tpc-ds   File: LocalExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void useDatabase(SessionContext session, String databaseName) throws SqlExecutionException {
	final ExecutionContext<?> context = getOrCreateExecutionContext(session);
	final TableEnvironment tableEnv = context
		.createEnvironmentInstance()
		.getTableEnvironment();

	context.wrapClassLoader(() -> {
		// Rely on TableEnvironment/CatalogManager to validate input
		try {
			tableEnv.useDatabase(databaseName);
		} catch (CatalogException e) {
			throw new SqlExecutionException("Failed to switch to database " + databaseName, e);
		}
		session.setCurrentDatabase(databaseName);
		return null;
	});
}
 
Example #2
Source Project: flink   Author: flink-tpc-ds   File: HiveShimV2.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> getViews(IMetaStoreClient client, String databaseName) throws UnknownDBException, TException {
	try {
		Method method = client.getClass().getMethod("getTables", String.class, String.class, TableType.class);
		return (List<String>) method.invoke(client, databaseName, null, TableType.VIRTUAL_VIEW);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format("Failed to get views for %s", databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format("Failed to get views for %s", databaseName), e);
	}
}
 
Example #3
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
private static HiveConf createHiveConf(@Nullable String hiveConfDir) {
	LOG.info("Setting hive conf dir as {}", hiveConfDir);

	try {
		HiveConf.setHiveSiteLocation(
			hiveConfDir == null ?
				null : Paths.get(hiveConfDir, "hive-site.xml").toUri().toURL());
	} catch (MalformedURLException e) {
		throw new CatalogException(
			String.format("Failed to get hive-site.xml from %s", hiveConfDir), e);
	}

	// create HiveConf from hadoop configuration
	return new HiveConf(HadoopUtils.getHadoopConfiguration(new org.apache.flink.configuration.Configuration()),
		HiveConf.class);
}
 
Example #4
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void alterTable(ObjectPath tablePath, CatalogBaseTable newTable, boolean ignoreIfNotExists)
		throws TableNotExistException {
	checkNotNull(tablePath);
	checkNotNull(newTable);

	CatalogBaseTable existingTable = tables.get(tablePath);

	if (existingTable != null) {
		if (existingTable.getClass() != newTable.getClass()) {
			throw new CatalogException(
				String.format("Table types don't match. Existing table is '%s' and new table is '%s'.",
					existingTable.getClass().getName(), newTable.getClass().getName()));
		}

		tables.put(tablePath, newTable.copy());
	} else if (!ignoreIfNotExists) {
		throw new TableNotExistException(getName(), tablePath);
	}
}
 
Example #5
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// pass -1 as max_parts to fetch all partitions
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), (short) -1).stream()
			.map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath, "Table path cannot be null");
	checkNotNull(partitionSpec, "CatalogPartitionSpec cannot be null");

	Table hiveTable = getHiveTable(tablePath);

	ensurePartitionedTable(tablePath, hiveTable);

	try {
		// partition spec can be partial
		List<String> partialVals = MetaStoreUtils.getPvals(hiveTable.getPartitionKeys(), partitionSpec.getPartitionSpec());
		return client.listPartitionNames(tablePath.getDatabaseName(), tablePath.getObjectName(), partialVals,
			(short) -1).stream().map(HiveCatalog::createPartitionSpec).collect(Collectors.toList());
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list partitions of table %s", tablePath), e);
	}
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<String> listFunctions(String databaseName) throws DatabaseNotExistException, CatalogException {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "databaseName cannot be null or empty");

	// client.getFunctions() returns empty list when the database doesn't exist
	// thus we need to explicitly check whether the database exists or not
	if (!databaseExists(databaseName)) {
		throw new DatabaseNotExistException(getName(), databaseName);
	}

	try {
		// hive-1.x requires the pattern not being null, so pass a pattern that matches any name
		return client.getFunctions(databaseName, ".*");
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list functions in database %s", databaseName), e);
	}
}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: HiveTypeUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TypeInfo visit(VarCharType varCharType) {
	// Flink's StringType is defined as VARCHAR(Integer.MAX_VALUE)
	// We don't have more information in LogicalTypeRoot to distinguish StringType and a VARCHAR(Integer.MAX_VALUE) instance
	// Thus always treat VARCHAR(Integer.MAX_VALUE) as StringType
	if (varCharType.getLength() == Integer.MAX_VALUE) {
		return TypeInfoFactory.stringTypeInfo;
	}
	if (varCharType.getLength() > HiveVarchar.MAX_VARCHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support varchar type with length of '%d'. " +
							"The maximum length is %d",
							varCharType.getLength(), HiveVarchar.MAX_VARCHAR_LENGTH));
	}
	return TypeInfoFactory.getVarcharTypeInfo(varCharType.getLength());
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: CatalogManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets the current database name that will be used when resolving a table path.
 * The database has to exist in the current catalog.
 *
 * @param databaseName database name to set as current database name
 * @throws CatalogException thrown if the database doesn't exist in the current catalog
 * @see CatalogManager#resolveTable(String...)
 * @see CatalogManager#setCurrentCatalog(String)
 */
public void setCurrentDatabase(String databaseName) {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "The database name cannot be null or empty.");

	if (!catalogs.get(currentCatalogName).databaseExists(databaseName)) {
		throw new CatalogException(format(
			"A database with name [%s] does not exist in the catalog: [%s].",
			databaseName,
			currentCatalogName));
	}

	if (!currentDatabaseName.equals(databaseName)) {
		currentDatabaseName = databaseName;

		LOG.info(
			"Set the current default database as [{}] in the current default catalog [{}].",
			currentCatalogName,
			currentDatabaseName);
	}
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void createPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogPartition partition, boolean ignoreIfExists)
		throws TableNotExistException, TableNotPartitionedException, PartitionSpecInvalidException, PartitionAlreadyExistsException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partition);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);
	ensureFullPartitionSpec(tablePath, partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		if (!ignoreIfExists) {
			throw new PartitionAlreadyExistsException(getName(), tablePath, partitionSpec);
		}
	}

	partitions.get(tablePath).put(partitionSpec, partition.copy());
}
 
Example #11
Source Project: pulsar-flink   Author: streamnative   File: PulsarCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws CatalogException {
    if (metadataReader != null) {
        metadataReader.close();
        metadataReader = null;
        log.info("Close connection to Pulsar");
    }
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: CatalogManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Registers a catalog under the given name. The catalog name must be unique across both
 * {@link Catalog}s and {@link ExternalCatalog}s.
 *
 * @param catalogName name under which to register the given catalog
 * @param catalog catalog to register
 * @throws CatalogException if the registration of the catalog under the given name failed
 */
public void registerCatalog(String catalogName, Catalog catalog) {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(catalogName), "Catalog name cannot be null or empty.");
	checkNotNull(catalog, "Catalog cannot be null");

	if (catalogs.containsKey(catalogName) || externalCatalogs.containsKey(catalogName)) {
		throw new CatalogException(format("Catalog %s already exists.", catalogName));
	}

	catalogs.put(catalogName, catalog);
	catalog.open();
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: HiveShimV1.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SimpleGenericUDAFParameterInfo createUDAFParameterInfo(ObjectInspector[] params, boolean isWindowing, boolean distinct, boolean allColumns) {
	try {
		Constructor constructor = SimpleGenericUDAFParameterInfo.class.getConstructor(ObjectInspector[].class,
				boolean.class, boolean.class);
		return (SimpleGenericUDAFParameterInfo) constructor.newInstance(params, distinct, allColumns);
	} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
		throw new CatalogException("Failed to create SimpleGenericUDAFParameterInfo", e);
	}
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: DatabaseCalciteSchema.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Set<String> getTableNames() {
	try {
		return new HashSet<>(catalog.listTables(databaseName));
	} catch (DatabaseNotExistException e) {
		throw new CatalogException(e);
	}
}
 
Example #15
Source Project: flink   Author: flink-tpc-ds   File: HiveShimV2.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SimpleGenericUDAFParameterInfo createUDAFParameterInfo(ObjectInspector[] params, boolean isWindowing, boolean distinct, boolean allColumns) {
	try {
		Constructor constructor = SimpleGenericUDAFParameterInfo.class.getConstructor(ObjectInspector[].class,
				boolean.class, boolean.class, boolean.class);
		return (SimpleGenericUDAFParameterInfo) constructor.newInstance(params, isWindowing, distinct, allColumns);
	} catch (NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) {
		throw new CatalogException("Failed to create SimpleGenericUDAFParameterInfo", e);
	}
}
 
Example #16
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open() throws CatalogException {
	if (client == null) {
		client = HiveMetastoreClientFactory.create(hiveConf, hiveVersion);
		LOG.info("Connected to Hive metastore");
	}

	if (!databaseExists(getDefaultDatabase())) {
		throw new CatalogException(String.format("Configured default database %s doesn't exist in catalog %s.",
			getDefaultDatabase(), getName()));
	}
}
 
Example #17
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws CatalogException {
	if (client != null) {
		client.close();
		client = null;
		LOG.info("Close connection to Hive metastore");
	}
}
 
Example #18
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void dropPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, boolean ignoreIfNotExists)
		throws PartitionNotExistException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (partitionExists(tablePath, partitionSpec)) {
		partitions.get(tablePath).remove(partitionSpec);
		partitionStats.get(tablePath).remove(partitionSpec);
		partitionColumnStats.get(tablePath).remove(partitionSpec);
	} else if (!ignoreIfNotExists) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}
}
 
Example #19
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<String> listDatabases() throws CatalogException {
	try {
		return client.getAllDatabases();
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to list all databases in %s", getName()), e);
	}
}
 
Example #20
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void renameTable(ObjectPath tablePath, String newTableName, boolean ignoreIfNotExists)
		throws TableNotExistException, TableAlreadyExistException, CatalogException {
	checkNotNull(tablePath, "tablePath cannot be null");
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(newTableName), "newTableName cannot be null or empty");

	try {
		// alter_table() doesn't throw a clear exception when target table doesn't exist.
		// Thus, check the table existence explicitly
		if (tableExists(tablePath)) {
			ObjectPath newPath = new ObjectPath(tablePath.getDatabaseName(), newTableName);
			// alter_table() doesn't throw a clear exception when new table already exists.
			// Thus, check the table existence explicitly
			if (tableExists(newPath)) {
				throw new TableAlreadyExistException(getName(), newPath);
			} else {
				Table table = getHiveTable(tablePath);
				table.setTableName(newTableName);
				client.alter_table(tablePath.getDatabaseName(), tablePath.getObjectName(), table);
			}
		} else if (!ignoreIfNotExists) {
			throw new TableNotExistException(getName(), tablePath);
		}
	} catch (TException e) {
		throw new CatalogException(
			String.format("Failed to rename table %s", tablePath.getFullName()), e);
	}
}
 
Example #21
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private static void ensureTableAndPartitionMatch(Table hiveTable, CatalogPartition catalogPartition) {
	boolean tableIsGeneric = Boolean.valueOf(hiveTable.getParameters().get(CatalogConfig.IS_GENERIC));
	boolean partitionIsGeneric = Boolean.valueOf(catalogPartition.getProperties().get(CatalogConfig.IS_GENERIC));

	if (tableIsGeneric != partitionIsGeneric) {
		throw new CatalogException(String.format("Cannot handle %s partition for %s table",
			catalogPartition.getClass().getName(), tableIsGeneric ? "generic" : "non-generic"));
	}
}
 
Example #22
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException,
		CatalogException {
	Table hiveTable = getHiveTable(tablePath);
	if (!isTablePartitioned(hiveTable)) {
		return createCatalogTableStatistics(hiveTable.getParameters());
	} else {
		return CatalogTableStatistics.UNKNOWN;
	}
}
 
Example #23
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CatalogPartition getPartition(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws PartitionNotExistException, CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (!partitionExists(tablePath, partitionSpec)) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}

	return partitions.get(tablePath).get(partitionSpec).copy();
}
 
Example #24
Source Project: flink   Author: flink-tpc-ds   File: HiveTypeUtil.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeInfo visit(CharType charType) {
	if (charType.getLength() > HiveChar.MAX_CHAR_LENGTH) {
		throw new CatalogException(
				String.format("HiveCatalog doesn't support char type with length of '%d'. " +
							"The maximum length is %d",
							charType.getLength(), HiveChar.MAX_CHAR_LENGTH));
	}
	return TypeInfoFactory.getCharTypeInfo(charType.getLength());
}
 
Example #25
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean partitionExists(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws CatalogException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	return partitions.containsKey(tablePath) && partitions.get(tablePath).containsKey(partitionSpec);
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: GenericInMemoryCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<CatalogPartitionSpec> listPartitions(ObjectPath tablePath)
		throws TableNotExistException, TableNotPartitionedException, CatalogException {
	checkNotNull(tablePath);

	ensureTableExists(tablePath);
	ensurePartitionedTable(tablePath);

	return new ArrayList<>(partitions.get(tablePath).keySet());
}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCharTypeLength() throws Exception {
	DataType[] types = new DataType[] {
		DataTypes.CHAR(HiveChar.MAX_CHAR_LENGTH + 1)
	};

	exception.expect(CatalogException.class);
	exception.expectMessage("HiveCatalog doesn't support char type with length of '256'. The maximum length is 255");
	verifyDataTypes(types);
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: DatabaseCalciteSchema.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Table getTable(String tableName) {

	ObjectPath tablePath = new ObjectPath(databaseName, tableName);

	try {
		if (!catalog.tableExists(tablePath)) {
			return null;
		}

		CatalogBaseTable table = catalog.getTable(tablePath);

		if (table instanceof QueryOperationCatalogView) {
			return QueryOperationCatalogViewTable.createCalciteTable(((QueryOperationCatalogView) table));
		} else if (table instanceof ConnectorCatalogTable) {
			return convertConnectorTable((ConnectorCatalogTable<?, ?>) table);
		} else if (table instanceof CatalogTable) {
			return convertCatalogTable(tablePath, (CatalogTable) table);
		} else {
			throw new TableException("Unsupported table type: " + table);
		}
	} catch (TableNotExistException | CatalogException e) {
		// TableNotExistException should never happen, because we are checking it exists
		// via catalog.tableExists
		throw new TableException(format(
			"A failure occurred when accessing table. Table path [%s, %s, %s]",
			catalogName,
			databaseName,
			tableName), e);
	}
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: DatabaseCalciteSchema.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Set<String> getTableNames() {
	try {
		return new HashSet<>(catalog.listTables(databaseName));
	} catch (DatabaseNotExistException e) {
		throw new CatalogException(e);
	}
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: CatalogManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRegisterCatalogWithExistingName() throws Exception {
	thrown.expect(CatalogException.class);

	CatalogManager manager = root()
		.builtin(
			database(BUILTIN_DEFAULT_DATABASE_NAME))
		.catalog(TEST_CATALOG_NAME, database(TEST_CATALOG_DEFAULT_DB_NAME))
		.build();

	manager.registerCatalog(TEST_CATALOG_NAME, new GenericInMemoryCatalog(TEST_CATALOG_NAME));
}