org.apache.flink.table.catalog.hive.descriptors.HiveCatalogValidator Java Examples

public HiveTableOutputFormat(JobConf jobConf, ObjectPath tablePath, CatalogTable table, HiveTablePartition hiveTablePartition,
							Properties tableProperties, boolean overwrite) {
	super(jobConf.getCredentials());

	Preconditions.checkNotNull(table, "table cannot be null");
	Preconditions.checkNotNull(hiveTablePartition, "HiveTablePartition cannot be null");
	Preconditions.checkNotNull(tableProperties, "Table properties cannot be null");

	HadoopUtils.mergeHadoopConf(jobConf);
	this.jobConf = jobConf;
	this.tablePath = tablePath;
	this.partitionColumns = table.getPartitionKeys();
	TableSchema tableSchema = table.getSchema();
	this.fieldNames = tableSchema.getFieldNames();
	this.fieldTypes = tableSchema.getFieldDataTypes();
	this.hiveTablePartition = hiveTablePartition;
	this.tableProperties = tableProperties;
	this.overwrite = overwrite;
	isPartitioned = partitionColumns != null && !partitionColumns.isEmpty();
	isDynamicPartition = isPartitioned && partitionColumns.size() > hiveTablePartition.getPartitionSpec().size();
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
}
 

@VisibleForTesting
protected HiveCatalog(String catalogName, String defaultDatabase, @Nullable HiveConf hiveConf, String hiveVersion,
		boolean allowEmbedded) {
	super(catalogName, defaultDatabase == null ? DEFAULT_DB : defaultDatabase);

	this.hiveConf = hiveConf == null ? createHiveConf(null) : hiveConf;
	if (!allowEmbedded) {
		checkArgument(!StringUtils.isNullOrWhitespaceOnly(this.hiveConf.getVar(HiveConf.ConfVars.METASTOREURIS)),
				"Embedded metastore is not allowed. Make sure you have set a valid value for " +
						HiveConf.ConfVars.METASTOREURIS.toString());
	}
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(hiveVersion), "hiveVersion cannot be null or empty");
	this.hiveVersion = hiveVersion;
	hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	// add this to hiveConf to make sure table factory and source/sink see the same Hive version as HiveCatalog
	this.hiveConf.set(HiveCatalogValidator.CATALOG_HIVE_VERSION, hiveVersion);

	LOG.info("Created HiveCatalog '{}'", catalogName);
}
 

@VisibleForTesting
protected HiveCatalog(String catalogName, String defaultDatabase, @Nullable HiveConf hiveConf, String hiveVersion) {
	super(catalogName, defaultDatabase == null ? DEFAULT_DB : defaultDatabase);

	this.hiveConf = hiveConf == null ? createHiveConf(null) : hiveConf;
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(hiveVersion), "hiveVersion cannot be null or empty");
	this.hiveVersion = hiveVersion;
	// add this to hiveConf to make sure table factory and source/sink see the same Hive version as HiveCatalog
	this.hiveConf.set(HiveCatalogValidator.CATALOG_HIVE_VERSION, hiveVersion);

	LOG.info("Created HiveCatalog '{}'", catalogName);
}
 

private static DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new HiveCatalogValidator().validate(descriptorProperties);

	return descriptorProperties;
}
 

public HiveTableSink(JobConf jobConf, ObjectPath tablePath, CatalogTable table) {
	this.jobConf = jobConf;
	this.tablePath = tablePath;
	this.catalogTable = table;
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
	tableSchema = table.getSchema();
}
 

public HiveTableFactory(HiveConf hiveConf) {
	this.hiveConf = checkNotNull(hiveConf, "hiveConf cannot be null");

	// this has to come from hiveConf, otherwise we may lose what user specifies in the yaml file
	this.hiveVersion = checkNotNull(hiveConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
}
 

public HiveTableSource(JobConf jobConf, ObjectPath tablePath, CatalogTable catalogTable) {
	this.jobConf = Preconditions.checkNotNull(jobConf);
	this.tablePath = Preconditions.checkNotNull(tablePath);
	this.catalogTable = Preconditions.checkNotNull(catalogTable);
	this.hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
	initAllPartitions = false;
	partitionPruned = false;
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	// Test HiveCatalogFactory.createCatalog
	// But not use it for testing purpose
	assertTrue(super.createCatalog(name, properties) != null);

	// Developers may already have their own production/testing hive-site.xml set in their environment,
	// and Flink tests should avoid using those hive-site.xml.
	// Thus, explicitly create a testing HiveConf for unit tests here
	Catalog hiveCatalog = HiveTestUtils.createHiveCatalog(name, properties.get(HiveCatalogValidator.CATALOG_HIVE_VERSION));

	// Creates an additional database to test tableEnv.useDatabase() will switch current database of the catalog
	hiveCatalog.open();
	try {
		hiveCatalog.createDatabase(
			ADDITIONAL_TEST_DATABASE,
			new CatalogDatabaseImpl(new HashMap<>(), null),
			false);
		hiveCatalog.createTable(
			new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
			new CatalogTableImpl(
				TableSchema.builder()
					.field("testcol", DataTypes.INT())
					.build(),
				new HashMap<String, String>() {{
					put(CatalogConfig.IS_GENERIC, String.valueOf(true));
				}},
				""
			),
			false
		);
	} catch (DatabaseAlreadyExistException | TableAlreadyExistException | DatabaseNotExistException e) {
		throw new CatalogException(e);
	}

	return hiveCatalog;
}
 

public HiveBatchSource(JobConf jobConf, ObjectPath tablePath, CatalogTable catalogTable) {
    this.jobConf = Preconditions.checkNotNull(jobConf);
    this.tablePath = Preconditions.checkNotNull(tablePath);
    this.catalogTable = Preconditions.checkNotNull(catalogTable);
    this.hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
        "Hive version is not defined");
    hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
    partitionPruned = false;
}
 

private static DescriptorProperties getValidatedProperties(Map<String, String> properties) {
	final DescriptorProperties descriptorProperties = new DescriptorProperties(true);
	descriptorProperties.putProperties(properties);

	new HiveCatalogValidator().validate(descriptorProperties);

	return descriptorProperties;
}
 

public HiveTableSink(
		boolean userMrWriter, boolean isBounded, JobConf jobConf, ObjectIdentifier identifier, CatalogTable table) {
	this.userMrWriter = userMrWriter;
	this.isBounded = isBounded;
	this.jobConf = jobConf;
	this.identifier = identifier;
	this.catalogTable = table;
	hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
	hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	tableSchema = TableSchemaUtils.getPhysicalSchema(table.getSchema());
}
 

public HiveTableSource(
		JobConf jobConf, ReadableConfig flinkConf, ObjectPath tablePath, CatalogTable catalogTable) {
	this.jobConf = Preconditions.checkNotNull(jobConf);
	this.flinkConf = Preconditions.checkNotNull(flinkConf);
	this.tablePath = Preconditions.checkNotNull(tablePath);
	this.catalogTable = Preconditions.checkNotNull(catalogTable);
	this.hiveVersion = Preconditions.checkNotNull(jobConf.get(HiveCatalogValidator.CATALOG_HIVE_VERSION),
			"Hive version is not defined");
	hiveShim = HiveShimLoader.loadHiveShim(hiveVersion);
	partitionPruned = false;
}
 

@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	// Developers may already have their own production/testing hive-site.xml set in their environment,
	// and Flink tests should avoid using those hive-site.xml.
	// Thus, explicitly create a testing HiveConf for unit tests here
	Catalog hiveCatalog = HiveTestUtils.createHiveCatalog(name, properties.get(HiveCatalogValidator.CATALOG_HIVE_VERSION));

	// Creates an additional database to test tableEnv.useDatabase() will switch current database of the catalog
	hiveCatalog.open();
	try {
		hiveCatalog.createDatabase(
			ADDITIONAL_TEST_DATABASE,
			new CatalogDatabaseImpl(new HashMap<>(), null),
			false);
		hiveCatalog.createTable(
			new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
			new CatalogTableImpl(
				TableSchema.builder()
					.field("testcol", DataTypes.INT())
					.build(),
				new HashMap<String, String>() {{
					put(CatalogConfig.IS_GENERIC, String.valueOf(false));
				}},
				""
			),
			false
		);
		// create a table to test parameterized types
		hiveCatalog.createTable(new ObjectPath("default", TABLE_WITH_PARAMETERIZED_TYPES),
				tableWithParameterizedTypes(),
				false);
	} catch (DatabaseAlreadyExistException | TableAlreadyExistException | DatabaseNotExistException e) {
		throw new CatalogException(e);
	}

	return hiveCatalog;
}