Java Code Examples for org.apache.flink.table.catalog.config.CatalogConfig

The following examples show how to use org.apache.flink.table.catalog.config.CatalogConfig. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
private static Function instantiateHiveFunction(ObjectPath functionPath, CatalogFunction function) {

		boolean isGeneric = Boolean.valueOf(function.getProperties().get(CatalogConfig.IS_GENERIC));

		// Hive Function does not have properties map
		// thus, use a prefix in class name to distinguish Flink and Hive functions
		String functionClassName = isGeneric ?
			FLINK_FUNCTION_PREFIX + function.getClassName() :
			function.getClassName();

		return new Function(
			// due to https://issues.apache.org/jira/browse/HIVE-22053, we have to normalize function name ourselves
			HiveStringUtils.normalizeIdentifier(functionPath.getObjectName()),
			functionPath.getDatabaseName(),
			functionClassName,
			null,			// Owner name
			PrincipalType.GROUP,	// Temporarily set to GROUP type because it's required by Hive. May change later
			(int) (System.currentTimeMillis() / 1000),
			FunctionType.JAVA,		// FunctionType only has JAVA now
			new ArrayList<>()		// Resource URIs
		);
	}
 
Example 2
Source Project: flink   Source File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 6 votes vote down vote up
private CatalogTable createCatalogTable(DataType[] types) {
	String[] colNames = new String[types.length];

	for (int i = 0; i < types.length; i++) {
		colNames[i] = String.format("%s_%d", types[i].toString().toLowerCase(), i);
	}

	TableSchema schema = TableSchema.builder()
		.fields(colNames, types)
		.build();

	return new CatalogTableImpl(
		schema,
		new HashMap<String, String>() {{
			put("is_streaming", "false");
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		""
	);
}
 
Example 3
Source Project: flink   Source File: HiveTableFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testGenericTable() throws Exception {
	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
	properties.put("connector", "COLLECTION");

	catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
	ObjectPath path = new ObjectPath("mydb", "mytable");
	CatalogTable table = new CatalogTableImpl(schema, properties, "csv table");
	catalog.createTable(path, table, true);
	Optional<TableFactory> opt = catalog.getTableFactory();
	assertTrue(opt.isPresent());
	HiveTableFactory tableFactory = (HiveTableFactory) opt.get();
	TableSource tableSource = tableFactory.createTableSource(path, table);
	assertTrue(tableSource instanceof StreamTableSource);
	TableSink tableSink = tableFactory.createTableSink(path, table);
	assertTrue(tableSink instanceof StreamTableSink);
}
 
Example 4
Source Project: flink   Source File: CatalogTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void checkEquals(CatalogTable t1, CatalogTable t2) {
	assertEquals(t1.getClass(), t2.getClass());
	assertEquals(t1.getSchema(), t2.getSchema());
	assertEquals(t1.getComment(), t2.getComment());
	assertEquals(t1.getPartitionKeys(), t2.getPartitionKeys());
	assertEquals(t1.isPartitioned(), t2.isPartitioned());

	assertEquals(
		t1.getProperties().get(CatalogConfig.IS_GENERIC),
		t2.getProperties().get(CatalogConfig.IS_GENERIC));

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(t1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(t1.getProperties(), t2.getProperties());
	} else {
		assertTrue(t2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(t2.getProperties().entrySet().containsAll(t1.getProperties().entrySet()));
	}
}
 
Example 5
Source Project: flink   Source File: CatalogTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void checkEquals(CatalogView v1, CatalogView v2) {
	assertEquals(v1.getClass(), v2.getClass());
	assertEquals(v1.getSchema(), v1.getSchema());
	assertEquals(v1.getComment(), v2.getComment());
	assertEquals(v1.getOriginalQuery(), v2.getOriginalQuery());
	assertEquals(v1.getExpandedQuery(), v2.getExpandedQuery());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(v1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(v1.getProperties(), v2.getProperties());
	} else {
		assertTrue(v2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(v2.getProperties().entrySet().containsAll(v1.getProperties().entrySet()));
	}
}
 
Example 6
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
public static boolean isGenericForCreate(Map<String, String> properties) {
	// When creating an object, a hive object needs explicitly have a key is_generic = false
	// otherwise, this is a generic object if 1) the key is missing 2) is_generic = true
	// this is opposite to reading an object. See getObjectIsGeneric().
	if (properties == null) {
		return true;
	}
	boolean isGeneric;
	if (!properties.containsKey(CatalogConfig.IS_GENERIC)) {
		// must be a generic object
		isGeneric = true;
		properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
	} else {
		isGeneric = Boolean.parseBoolean(properties.get(CatalogConfig.IS_GENERIC));
	}
	return isGeneric;
}
 
Example 7
Source Project: flink   Source File: HiveTableFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TableSource<RowData> createTableSource(TableSourceFactory.Context context) {
	CatalogTable table = checkNotNull(context.getTable());
	Preconditions.checkArgument(table instanceof CatalogTableImpl);

	boolean isGeneric = Boolean.parseBoolean(table.getProperties().get(CatalogConfig.IS_GENERIC));

	if (!isGeneric) {
		return new HiveTableSource(
				new JobConf(hiveConf),
				context.getConfiguration(),
				context.getObjectIdentifier().toObjectPath(),
				table);
	} else {
		return TableFactoryUtil.findAndCreateTableSource(context);
	}
}
 
Example 8
Source Project: flink   Source File: HiveTableFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TableSink createTableSink(TableSinkFactory.Context context) {
	CatalogTable table = checkNotNull(context.getTable());
	Preconditions.checkArgument(table instanceof CatalogTableImpl);

	boolean isGeneric = Boolean.parseBoolean(table.getProperties().get(CatalogConfig.IS_GENERIC));

	if (!isGeneric) {
		return new HiveTableSink(
				context.getConfiguration().get(
						HiveOptions.TABLE_EXEC_HIVE_FALLBACK_MAPRED_WRITER),
				context.isBounded(),
				new JobConf(hiveConf),
				context.getObjectIdentifier(),
				table);
	} else {
		return TableFactoryUtil.findAndCreateTableSink(context);
	}
}
 
Example 9
Source Project: flink   Source File: HiveCatalogTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateHiveTable() {
	Map<String, String> map = new HashMap<>(new FileSystem().path("/test_path").toProperties());

	map.put(CatalogConfig.IS_GENERIC, String.valueOf(false));

	Table hiveTable = HiveTableUtil.instantiateHiveTable(
		new ObjectPath("test", "test"),
		new CatalogTableImpl(
			schema,
			map,
			null
		),
		HiveTestUtils.createHiveConf());

	Map<String, String> prop = hiveTable.getParameters();
	assertEquals(prop.remove(CatalogConfig.IS_GENERIC), String.valueOf(false));
	assertTrue(prop.keySet().stream().noneMatch(k -> k.startsWith(CatalogConfig.FLINK_PROPERTY_PREFIX)));
}
 
Example 10
Source Project: flink   Source File: HiveCatalogHiveMetadataTest.java    License: Apache License 2.0 6 votes vote down vote up
private void checkStatistics(int inputStat, int expectStat) throws Exception {
	catalog.dropTable(path1, true);

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, "false");
	properties.put(StatsSetupConst.ROW_COUNT, String.valueOf(inputStat));
	properties.put(StatsSetupConst.NUM_FILES, String.valueOf(inputStat));
	properties.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(inputStat));
	properties.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(inputStat));
	CatalogTable catalogTable = new CatalogTableImpl(
			TableSchema.builder().field("f0", DataTypes.INT()).build(),
			properties,
			"");
	catalog.createTable(path1, catalogTable, false);

	CatalogTableStatistics statistics = catalog.getTableStatistics(path1);
	assertEquals(expectStat, statistics.getRowCount());
	assertEquals(expectStat, statistics.getFileCount());
	assertEquals(expectStat, statistics.getRawDataSize());
	assertEquals(expectStat, statistics.getTotalSize());
}
 
Example 11
Source Project: flink   Source File: HiveCatalogDataTypeTest.java    License: Apache License 2.0 6 votes vote down vote up
private CatalogTable createCatalogTable(DataType[] types) {
	String[] colNames = new String[types.length];

	for (int i = 0; i < types.length; i++) {
		colNames[i] = String.format("%s_%d", types[i].toString().toLowerCase(), i);
	}

	TableSchema schema = TableSchema.builder()
		.fields(colNames, types)
		.build();

	return new CatalogTableImpl(
		schema,
		new HashMap<String, String>() {{
			put("is_streaming", "false");
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		""
	);
}
 
Example 12
Source Project: flink   Source File: HiveTableSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
private CatalogTable createHiveCatalogTable(TableSchema tableSchema, int numPartCols) {
	if (numPartCols == 0) {
		return new CatalogTableImpl(
			tableSchema,
			new HashMap<String, String>() {{
				// creating a hive table needs explicit is_generic=false flag
				put(CatalogConfig.IS_GENERIC, String.valueOf(false));
			}},
			"");
	}
	String[] partCols = new String[numPartCols];
	System.arraycopy(tableSchema.getFieldNames(), tableSchema.getFieldNames().length - numPartCols, partCols, 0, numPartCols);
	return new CatalogTableImpl(
		tableSchema,
		Arrays.asList(partCols),
		new HashMap<String, String>() {{
			// creating a hive table needs explicit is_generic=false flag
			put(CatalogConfig.IS_GENERIC, String.valueOf(false));
		}},
		"");
}
 
Example 13
Source Project: flink   Source File: SqlCreateHiveView.java    License: Apache License 2.0 6 votes vote down vote up
public SqlCreateHiveView(SqlParserPos pos, SqlIdentifier viewName, SqlNodeList fieldList, SqlNode query,
		boolean ifNotExists, SqlCharStringLiteral comment, SqlNodeList properties) {
	super(
			pos,
			viewName,
			fieldList,
			query,
			false,
			false,
			ifNotExists,
			HiveDDLUtils.unescapeStringLiteral(comment),
			properties
	);
	HiveDDLUtils.unescapeProperties(properties);
	originPropList = new SqlNodeList(properties.getList(), properties.getParserPosition());
	// mark it as a hive view
	properties.add(HiveDDLUtils.toTableOption(CatalogConfig.IS_GENERIC, "false", pos));
}
 
Example 14
Source Project: flink   Source File: SqlCreateHiveDatabase.java    License: Apache License 2.0 6 votes vote down vote up
public SqlCreateHiveDatabase(SqlParserPos pos, SqlIdentifier databaseName, SqlNodeList propertyList,
		SqlCharStringLiteral comment, SqlCharStringLiteral location, boolean ifNotExists) throws ParseException {
	super(
			pos,
			databaseName,
			HiveDDLUtils.checkReservedDBProperties(propertyList),
			HiveDDLUtils.unescapeStringLiteral(comment),
			ifNotExists
	);
	HiveDDLUtils.ensureNonGeneric(propertyList);
	originPropList = new SqlNodeList(propertyList.getList(), propertyList.getParserPosition());
	// mark it as a hive database
	propertyList.add(HiveDDLUtils.toTableOption(CatalogConfig.IS_GENERIC, "false", pos));
	if (location != null) {
		propertyList.add(new SqlTableOption(
				SqlLiteral.createCharString(DATABASE_LOCATION_URI, location.getParserPosition()),
				location,
				location.getParserPosition()));
	}
	this.location = location;
}
 
Example 15
Source Project: flink   Source File: CatalogTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void checkEquals(CatalogTable t1, CatalogTable t2) {
	assertEquals(t1.getClass(), t2.getClass());
	assertEquals(t1.getSchema(), t2.getSchema());
	assertEquals(t1.getComment(), t2.getComment());
	assertEquals(t1.getPartitionKeys(), t2.getPartitionKeys());
	assertEquals(t1.isPartitioned(), t2.isPartitioned());

	assertEquals(
		t1.getProperties().get(CatalogConfig.IS_GENERIC),
		t2.getProperties().get(CatalogConfig.IS_GENERIC));

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(t1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(t1.getProperties(), t2.getProperties());
	} else {
		assertTrue(t2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(t2.getProperties().entrySet().containsAll(t1.getProperties().entrySet()));
	}
}
 
Example 16
Source Project: flink   Source File: CatalogTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void checkEquals(CatalogView v1, CatalogView v2) {
	assertEquals(v1.getClass(), v2.getClass());
	assertEquals(v1.getSchema(), v1.getSchema());
	assertEquals(v1.getComment(), v2.getComment());
	assertEquals(v1.getOriginalQuery(), v2.getOriginalQuery());
	assertEquals(v1.getExpandedQuery(), v2.getExpandedQuery());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(v1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(v1.getProperties(), v2.getProperties());
	} else {
		assertTrue(v2.getProperties().keySet().stream().noneMatch(k -> k.startsWith(FLINK_PROPERTY_PREFIX)));
		assertTrue(v2.getProperties().entrySet().containsAll(v1.getProperties().entrySet()));
	}
}
 
Example 17
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add a prefix to Flink-created properties to distinguish them from Hive-created properties.
 * Note that 'is_generic' is a special key and this method will leave it as-is.
 */
private static Map<String, String> maskFlinkProperties(Map<String, String> properties) {
	return properties.entrySet().stream()
		.filter(e -> e.getKey() != null && e.getValue() != null)
		.map(e -> new Tuple2<>(
			e.getKey().equals(CatalogConfig.IS_GENERIC) ? e.getKey() : FLINK_PROPERTY_PREFIX + e.getKey(),
			e.getValue()))
		.collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 
Example 18
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private static void ensureTableAndPartitionMatch(Table hiveTable, CatalogPartition catalogPartition) {
	boolean tableIsGeneric = Boolean.valueOf(hiveTable.getParameters().get(CatalogConfig.IS_GENERIC));
	boolean partitionIsGeneric = Boolean.valueOf(catalogPartition.getProperties().get(CatalogConfig.IS_GENERIC));

	if (tableIsGeneric != partitionIsGeneric) {
		throw new CatalogException(String.format("Cannot handle %s partition for %s table",
			catalogPartition.getClass().getName(), tableIsGeneric ? "generic" : "non-generic"));
	}
}
 
Example 19
Source Project: flink   Source File: HiveTableFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TableSource<Row> createTableSource(ObjectPath tablePath, CatalogTable table) {
	Preconditions.checkNotNull(table);
	Preconditions.checkArgument(table instanceof CatalogTableImpl);

	boolean isGeneric = Boolean.valueOf(table.getProperties().get(CatalogConfig.IS_GENERIC));

	if (!isGeneric) {
		return createInputFormatTableSource(tablePath, table);
	} else {
		return TableFactoryUtil.findAndCreateTableSource(table);
	}
}
 
Example 20
Source Project: flink   Source File: HiveTableFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TableSink<Row> createTableSink(ObjectPath tablePath, CatalogTable table) {
	Preconditions.checkNotNull(table);
	Preconditions.checkArgument(table instanceof CatalogTableImpl);

	boolean isGeneric = Boolean.valueOf(table.getProperties().get(CatalogConfig.IS_GENERIC));

	if (!isGeneric) {
		return createOutputFormatTableSink(tablePath, table);
	} else {
		return TableFactoryUtil.findAndCreateTableSink(table);
	}
}
 
Example 21
Source Project: flink   Source File: CatalogTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static void checkEquals(CatalogPartition p1, CatalogPartition p2) {
	assertEquals(p1.getClass(), p2.getClass());
	assertEquals(p1.getComment(), p2.getComment());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(p1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(p1.getProperties(), p2.getProperties());
	} else {
		assertTrue(p2.getProperties().entrySet().containsAll(p1.getProperties().entrySet()));
	}
}
 
Example 22
Source Project: flink   Source File: CatalogTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CatalogTable createStreamingTable() {
	Map<String, String> prop = getBatchTableProperties();
	prop.put(CatalogConfig.IS_GENERIC, String.valueOf(false));

	return new CatalogTableImpl(
		createTableSchema(),
		getStreamingTableProperties(),
		TEST_COMMENT);
}
 
Example 23
Source Project: flink   Source File: DependencyTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Catalog createCatalog(String name, Map<String, String> properties) {
	// Test HiveCatalogFactory.createCatalog
	// But not use it for testing purpose
	assertTrue(super.createCatalog(name, properties) != null);

	// Developers may already have their own production/testing hive-site.xml set in their environment,
	// and Flink tests should avoid using those hive-site.xml.
	// Thus, explicitly create a testing HiveConf for unit tests here
	Catalog hiveCatalog = HiveTestUtils.createHiveCatalog(name, properties.get(HiveCatalogValidator.CATALOG_HIVE_VERSION));

	// Creates an additional database to test tableEnv.useDatabase() will switch current database of the catalog
	hiveCatalog.open();
	try {
		hiveCatalog.createDatabase(
			ADDITIONAL_TEST_DATABASE,
			new CatalogDatabaseImpl(new HashMap<>(), null),
			false);
		hiveCatalog.createTable(
			new ObjectPath(ADDITIONAL_TEST_DATABASE, TEST_TABLE),
			new CatalogTableImpl(
				TableSchema.builder()
					.field("testcol", DataTypes.INT())
					.build(),
				new HashMap<String, String>() {{
					put(CatalogConfig.IS_GENERIC, String.valueOf(true));
				}},
				""
			),
			false
		);
	} catch (DatabaseAlreadyExistException | TableAlreadyExistException | DatabaseNotExistException e) {
		throw new CatalogException(e);
	}

	return hiveCatalog;
}
 
Example 24
Source Project: flink   Source File: HiveTableUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add a prefix to Flink-created properties to distinguish them from Hive-created properties.
 * Note that 'is_generic' is a special key and this method will leave it as-is.
 */
public static Map<String, String> maskFlinkProperties(Map<String, String> properties) {
	return properties.entrySet().stream()
			.filter(e -> e.getKey() != null && e.getValue() != null)
			.map(e -> new Tuple2<>(
					e.getKey().equals(CatalogConfig.IS_GENERIC) ? e.getKey() : FLINK_PROPERTY_PREFIX + e.getKey(),
					e.getValue()))
			.collect(Collectors.toMap(t -> t.f0, t -> t.f1));
}
 
Example 25
Source Project: flink   Source File: HiveCatalogTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateGenericTable() {
	Table hiveTable = HiveTableUtil.instantiateHiveTable(
		new ObjectPath("test", "test"),
		new CatalogTableImpl(
			schema,
			new FileSystem().path("/test_path").toProperties(),
			null
		),
		HiveTestUtils.createHiveConf());

	Map<String, String> prop = hiveTable.getParameters();
	assertEquals(prop.remove(CatalogConfig.IS_GENERIC), String.valueOf("true"));
	assertTrue(prop.keySet().stream().allMatch(k -> k.startsWith(CatalogConfig.FLINK_PROPERTY_PREFIX)));
}
 
Example 26
Source Project: flink   Source File: HiveDialectITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateDatabase() throws Exception {
	tableEnv.executeSql("create database db1 comment 'db1 comment'");
	Database db = hiveCatalog.getHiveDatabase("db1");
	assertEquals("db1 comment", db.getDescription());
	assertFalse(Boolean.parseBoolean(db.getParameters().get(CatalogConfig.IS_GENERIC)));

	String db2Location = warehouse + "/db2_location";
	tableEnv.executeSql(String.format("create database db2 location '%s' with dbproperties('k1'='v1')", db2Location));
	db = hiveCatalog.getHiveDatabase("db2");
	assertEquals(db2Location, locationPath(db.getLocationUri()));
	assertEquals("v1", db.getParameters().get("k1"));
}
 
Example 27
Source Project: flink   Source File: HiveTableFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGenericTable() throws Exception {
	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, String.valueOf(true));
	properties.put("connector", "COLLECTION");

	catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
	ObjectPath path = new ObjectPath("mydb", "mytable");
	CatalogTable table = new CatalogTableImpl(schema, properties, "csv table");
	catalog.createTable(path, table, true);
	Optional<TableFactory> opt = catalog.getTableFactory();
	assertTrue(opt.isPresent());
	HiveTableFactory tableFactory = (HiveTableFactory) opt.get();
	TableSource tableSource = tableFactory.createTableSource(new TableSourceFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"), table, new Configuration()));
	assertTrue(tableSource instanceof StreamTableSource);
	TableSink tableSink = tableFactory.createTableSink(new TableSinkFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"),
			table,
			new Configuration(),
			true));
	assertTrue(tableSink instanceof StreamTableSink);
}
 
Example 28
Source Project: flink   Source File: HiveTableFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHiveTable() throws Exception {
	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, String.valueOf(false));

	catalog.createDatabase("mydb", new CatalogDatabaseImpl(new HashMap<>(), ""), true);
	ObjectPath path = new ObjectPath("mydb", "mytable");
	CatalogTable table = new CatalogTableImpl(schema, properties, "hive table");
	catalog.createTable(path, table, true);
	Optional<TableFactory> opt = catalog.getTableFactory();
	assertTrue(opt.isPresent());
	HiveTableFactory tableFactory = (HiveTableFactory) opt.get();
	TableSink tableSink = tableFactory.createTableSink(new TableSinkFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"),
			table,
			new Configuration(),
			true));
	assertTrue(tableSink instanceof HiveTableSink);
	TableSource tableSource = tableFactory.createTableSource(new TableSourceFactoryContextImpl(
			ObjectIdentifier.of("mycatalog", "mydb", "mytable"), table, new Configuration()));
	assertTrue(tableSource instanceof HiveTableSource);
}
 
Example 29
Source Project: flink   Source File: HiveDDLUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static SqlNodeList ensureNonGeneric(SqlNodeList props) throws ParseException {
	for (SqlNode node : props) {
		if (node instanceof SqlTableOption && ((SqlTableOption) node).getKeyString().equalsIgnoreCase(CatalogConfig.IS_GENERIC)) {
			if (!((SqlTableOption) node).getValueString().equalsIgnoreCase("false")) {
				throw new ParseException("Creating generic object with Hive dialect is not allowed");
			}
		}
	}
	return props;
}
 
Example 30
Source Project: flink   Source File: CatalogTestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static void checkEquals(CatalogPartition p1, CatalogPartition p2) {
	assertEquals(p1.getClass(), p2.getClass());
	assertEquals(p1.getComment(), p2.getComment());

	// Hive tables may have properties created by itself
	// thus properties of Hive table is a super set of those in its corresponding Flink table
	if (Boolean.valueOf(p1.getProperties().get(CatalogConfig.IS_GENERIC))) {
		assertEquals(p1.getProperties(), p2.getProperties());
	} else {
		assertTrue(p2.getProperties().entrySet().containsAll(p1.getProperties().entrySet()));
	}
}