org.apache.flink.table.descriptors.FormatDescriptor Java Examples

The following examples show how to use org.apache.flink.table.descriptors.FormatDescriptor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: HiveCatalogITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testGenericTable() throws Exception {
	ExecutionEnvironment execEnv = ExecutionEnvironment.createLocalEnvironment(1);
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(execEnv);

	tableEnv.registerCatalog("myhive", hiveCatalog);

	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	FormatDescriptor format = new OldCsv()
		.field("name", Types.STRING())
		.field("age", Types.INT());

	CatalogTable source =
		new CatalogTableBuilder(
			new FileSystem().path(this.getClass().getResource("/csv/test.csv").getPath()),
			schema)
		.withFormat(format)
		.inAppendMode()
		.withComment("Comment.")
		.build();

	Path p = Paths.get(tempFolder.newFolder().getAbsolutePath(), "test.csv");

	CatalogTable sink =
		new CatalogTableBuilder(
			new FileSystem().path(p.toAbsolutePath().toString()),
			schema)
			.withFormat(format)
			.inAppendMode()
			.withComment("Comment.")
			.build();

	hiveCatalog.createTable(
		new ObjectPath(HiveCatalog.DEFAULT_DB, sourceTableName),
		source,
		false
	);

	hiveCatalog.createTable(
		new ObjectPath(HiveCatalog.DEFAULT_DB, sinkTableName),
		sink,
		false
	);

	Table t = tableEnv.sqlQuery(
		String.format("select * from myhive.`default`.%s", sourceTableName));

	List<Row> result = tableEnv.toDataSet(t, Row.class).collect();

	// assert query result
	assertEquals(
		Arrays.asList(
			Row.of("1", 1),
			Row.of("2", 2),
			Row.of("3", 3)),
		result
	);

	tableEnv.sqlUpdate(
		String.format("insert into myhive.`default`.%s select * from myhive.`default`.%s",
			sinkTableName,
			sourceTableName));
	tableEnv.execute("myjob");

	// assert written result
	File resultFile = new File(p.toAbsolutePath().toString());
	BufferedReader reader = new BufferedReader(new FileReader(resultFile));
	String readLine;
	for (int i = 0; i < 3; i++) {
		readLine = reader.readLine();
		assertEquals(String.format("%d,%d", i + 1, i + 1), readLine);
	}

	// No more line
	assertNull(reader.readLine());
}

Example #2

Source File: FlinkPravegaTableSourceTest.java From flink-connectors with Apache License 2.0

4 votes

@Override
public TestTableDescriptor withFormat(FormatDescriptor format) {
    super.withFormat(format);
    return this;
}

Example #3

Source File: HiveCatalogITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testCsvTableViaAPI() throws Exception {
	EnvironmentSettings settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build();
	TableEnvironment tableEnv = TableEnvironment.create(settings);
	tableEnv.getConfig().addConfiguration(new Configuration().set(CoreOptions.DEFAULT_PARALLELISM, 1));

	tableEnv.registerCatalog("myhive", hiveCatalog);
	tableEnv.useCatalog("myhive");

	TableSchema schema = TableSchema.builder()
		.field("name", DataTypes.STRING())
		.field("age", DataTypes.INT())
		.build();

	FormatDescriptor format = new OldCsv()
		.field("name", Types.STRING())
		.field("age", Types.INT());

	CatalogTable source =
		new CatalogTableBuilder(
			new FileSystem().path(this.getClass().getResource("/csv/test.csv").getPath()),
			schema)
		.withFormat(format)
		.inAppendMode()
		.withComment("Comment.")
		.build();

	Path p = Paths.get(tempFolder.newFolder().getAbsolutePath(), "test.csv");

	CatalogTable sink =
		new CatalogTableBuilder(
			new FileSystem().path(p.toAbsolutePath().toString()),
			schema)
			.withFormat(format)
			.inAppendMode()
			.withComment("Comment.")
			.build();

	hiveCatalog.createTable(
		new ObjectPath(HiveCatalog.DEFAULT_DB, sourceTableName),
		source,
		false
	);

	hiveCatalog.createTable(
		new ObjectPath(HiveCatalog.DEFAULT_DB, sinkTableName),
		sink,
		false
	);

	Table t = tableEnv.sqlQuery(
		String.format("select * from myhive.`default`.%s", sourceTableName));

	List<Row> result = Lists.newArrayList(t.execute().collect());
	result.sort(Comparator.comparing(String::valueOf));

	// assert query result
	assertEquals(
		Arrays.asList(
			Row.of("1", 1),
			Row.of("2", 2),
			Row.of("3", 3)),
		result
	);

	TableEnvUtil.execInsertSqlAndWaitResult(tableEnv,
		String.format("insert into myhive.`default`.%s select * from myhive.`default`.%s",
			sinkTableName,
			sourceTableName));

	// assert written result
	File resultFile = new File(p.toAbsolutePath().toString());
	BufferedReader reader = new BufferedReader(new FileReader(resultFile));
	String readLine;
	for (int i = 0; i < 3; i++) {
		readLine = reader.readLine();
		assertEquals(String.format("%d,%d", i + 1, i + 1), readLine);
	}

	// No more line
	assertNull(reader.readLine());

	tableEnv.executeSql(String.format("DROP TABLE %s", sourceTableName));
	tableEnv.executeSql(String.format("DROP TABLE %s", sinkTableName));
}

Example #4

Source File: HiveCatalogUseBlinkITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testBlinkUdf() throws Exception {
	TableSchema schema = TableSchema.builder()
			.field("name", DataTypes.STRING())
			.field("age", DataTypes.INT())
			.build();

	FormatDescriptor format = new OldCsv()
			.field("name", Types.STRING())
			.field("age", Types.INT());

	CatalogTable source =
			new CatalogTableBuilder(
					new FileSystem().path(this.getClass().getResource("/csv/test.csv").getPath()),
					schema)
					.withFormat(format)
					.inAppendMode()
					.withComment("Comment.")
					.build();

	hiveCatalog.createTable(
			new ObjectPath(HiveCatalog.DEFAULT_DB, sourceTableName),
			source,
			false
	);

	hiveCatalog.createFunction(
			new ObjectPath(HiveCatalog.DEFAULT_DB, "myudf"),
			new CatalogFunctionImpl(TestHiveSimpleUDF.class.getCanonicalName()),
			false);
	hiveCatalog.createFunction(
			new ObjectPath(HiveCatalog.DEFAULT_DB, "mygenericudf"),
			new CatalogFunctionImpl(TestHiveGenericUDF.class.getCanonicalName()),
			false);
	hiveCatalog.createFunction(
			new ObjectPath(HiveCatalog.DEFAULT_DB, "myudtf"),
			new CatalogFunctionImpl(TestHiveUDTF.class.getCanonicalName()),
			false);
	hiveCatalog.createFunction(
			new ObjectPath(HiveCatalog.DEFAULT_DB, "myudaf"),
			new CatalogFunctionImpl(GenericUDAFSum.class.getCanonicalName()),
			false);

	testUdf(true);
	testUdf(false);
}