org.apache.flink.table.catalog.stats.CatalogColumnStatistics Java Examples

The following examples show how to use org.apache.flink.table.catalog.stats.CatalogColumnStatistics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: HiveCatalog.java From flink with Apache License 2.0

6 votes

@Override
public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException {
	Table hiveTable = getHiveTable(tablePath);
	try {
		if (!isTablePartitioned(hiveTable)) {
			List<ColumnStatisticsObj> columnStatisticsObjs = client.getTableColumnStatistics(
					hiveTable.getDbName(), hiveTable.getTableName(), getFieldNames(hiveTable.getSd().getCols()));
			return new CatalogColumnStatistics(HiveStatsUtil.createCatalogColumnStats(columnStatisticsObjs, hiveVersion));
		} else {
			// TableColumnStats of partitioned table is unknown, the behavior is same as HIVE
			return CatalogColumnStatistics.UNKNOWN;
		}
	} catch (TException e) {
		throw new CatalogException(String.format("Failed to get table column stats of table %s",
												tablePath.getFullName()), e);
	}
}

Example #2

Source File: GenericInMemoryCatalogTest.java From flink with Apache License 2.0

6 votes

private CatalogColumnStatistics createColumnStats() {
	CatalogColumnStatisticsDataBoolean booleanColStats = new CatalogColumnStatisticsDataBoolean(55L, 45L, 5L);
	CatalogColumnStatisticsDataLong longColStats = new CatalogColumnStatisticsDataLong(-123L, 763322L, 23L, 79L);
	CatalogColumnStatisticsDataString stringColStats = new CatalogColumnStatisticsDataString(152L, 43.5D, 20L, 0L);
	CatalogColumnStatisticsDataDate dateColStats = new CatalogColumnStatisticsDataDate(new Date(71L),
		new Date(17923L), 1321, 0L);
	CatalogColumnStatisticsDataDouble doubleColStats = new CatalogColumnStatisticsDataDouble(-123.35D, 7633.22D, 23L, 79L);
	CatalogColumnStatisticsDataBinary binaryColStats = new CatalogColumnStatisticsDataBinary(755L, 43.5D, 20L);
	Map<String, CatalogColumnStatisticsDataBase> colStatsMap = new HashMap<>(6);
	colStatsMap.put("b1", booleanColStats);
	colStatsMap.put("l2", longColStats);
	colStatsMap.put("s3", stringColStats);
	colStatsMap.put("d4", dateColStats);
	colStatsMap.put("dd5", doubleColStats);
	colStatsMap.put("bb6", binaryColStats);
	return new CatalogColumnStatistics(colStatsMap);
}

Example #3

Source File: CatalogTableStatisticsConverter.java From flink with Apache License 2.0

6 votes

public static TableStats convertToTableStats(
		CatalogTableStatistics tableStatistics,
		CatalogColumnStatistics columnStatistics) {
	if (tableStatistics == null || tableStatistics.equals(CatalogTableStatistics.UNKNOWN)) {
		return TableStats.UNKNOWN;
	}

	long rowCount = tableStatistics.getRowCount();
	Map<String, ColumnStats> columnStatsMap = null;
	if (columnStatistics != null && !columnStatistics.equals(CatalogColumnStatistics.UNKNOWN)) {
		columnStatsMap = convertToColumnStatsMap(columnStatistics.getColumnStatisticsData());
	}
	if (columnStatsMap == null) {
		columnStatsMap = new HashMap<>();
	}
	return new TableStats(rowCount, columnStatsMap);
}

Example #4

Source File: CatalogStatisticsTest.java From flink with Apache License 2.0

6 votes

private CatalogColumnStatistics createColumnStats() {
	CatalogColumnStatisticsDataBoolean booleanColStats = new CatalogColumnStatisticsDataBoolean(55L, 45L, 5L);
	CatalogColumnStatisticsDataLong longColStats = new CatalogColumnStatisticsDataLong(-123L, 763322L, 23L, 79L);
	CatalogColumnStatisticsDataString stringColStats = new CatalogColumnStatisticsDataString(152L, 43.5D, 20L, 0L);
	CatalogColumnStatisticsDataDate dateColStats =
			new CatalogColumnStatisticsDataDate(new Date(71L), new Date(17923L), 1321, 0L);
	CatalogColumnStatisticsDataDouble doubleColStats =
			new CatalogColumnStatisticsDataDouble(-123.35D, 7633.22D, 23L, 79L);
	Map<String, CatalogColumnStatisticsDataBase> colStatsMap = new HashMap<>(6);
	colStatsMap.put("b1", booleanColStats);
	colStatsMap.put("l2", longColStats);
	colStatsMap.put("s3", stringColStats);
	colStatsMap.put("d4", dateColStats);
	colStatsMap.put("dd5", doubleColStats);
	return new CatalogColumnStatistics(colStatsMap);
}

Example #5

Source File: GenericInMemoryCatalogTest.java From flink with Apache License 2.0

6 votes

private CatalogColumnStatistics createColumnStats() {
	CatalogColumnStatisticsDataBoolean booleanColStats = new CatalogColumnStatisticsDataBoolean(55L, 45L, 5L);
	CatalogColumnStatisticsDataLong longColStats = new CatalogColumnStatisticsDataLong(-123L, 763322L, 23L, 79L);
	CatalogColumnStatisticsDataString stringColStats = new CatalogColumnStatisticsDataString(152L, 43.5D, 20L, 0L);
	CatalogColumnStatisticsDataDate dateColStats = new CatalogColumnStatisticsDataDate(new Date(71L),
		new Date(17923L), 1321L, 0L);
	CatalogColumnStatisticsDataDouble doubleColStats = new CatalogColumnStatisticsDataDouble(-123.35D, 7633.22D, 23L, 79L);
	CatalogColumnStatisticsDataBinary binaryColStats = new CatalogColumnStatisticsDataBinary(755L, 43.5D, 20L);
	Map<String, CatalogColumnStatisticsDataBase> colStatsMap = new HashMap<>(6);
	colStatsMap.put("b1", booleanColStats);
	colStatsMap.put("l2", longColStats);
	colStatsMap.put("s3", stringColStats);
	colStatsMap.put("d4", dateColStats);
	colStatsMap.put("dd5", doubleColStats);
	colStatsMap.put("bb6", binaryColStats);
	return new CatalogColumnStatistics(colStatsMap);
}

Example #6

Source File: DatabaseCalciteSchema.java From flink with Apache License 2.0

6 votes

private static TableStats extractTableStats(
		Catalog catalog,
		ObjectIdentifier objectIdentifier) {
	final ObjectPath tablePath = objectIdentifier.toObjectPath();
	try {
		CatalogTableStatistics tableStatistics = catalog.getTableStatistics(tablePath);
		CatalogColumnStatistics columnStatistics = catalog.getTableColumnStatistics(tablePath);
		return convertToTableStats(tableStatistics, columnStatistics);
	} catch (TableNotExistException e) {
		throw new ValidationException(format(
			"Could not get statistic for table: [%s, %s, %s]",
			objectIdentifier.getCatalogName(),
			tablePath.getDatabaseName(),
			tablePath.getObjectName()), e);
	}
}

Example #7

Source File: CatalogTableStatisticsConverter.java From flink with Apache License 2.0

6 votes

public static TableStats convertToTableStats(
		CatalogTableStatistics tableStatistics,
		CatalogColumnStatistics columnStatistics) {
	long rowCount;
	if (tableStatistics != null && tableStatistics.getRowCount() >= 0) {
		rowCount = tableStatistics.getRowCount();
	} else {
		rowCount = TableStats.UNKNOWN.getRowCount();
	}

	Map<String, ColumnStats> columnStatsMap;
	if (columnStatistics != null) {
		columnStatsMap = convertToColumnStatsMap(columnStatistics.getColumnStatisticsData());
	} else {
		columnStatsMap = new HashMap<>();
	}
	return new TableStats(rowCount, columnStatsMap);
}

Example #8

Source File: HiveCatalog.java From flink with Apache License 2.0

6 votes

@Override
public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException, CatalogException {
	Table hiveTable = getHiveTable(tablePath);
	try {
		if (!isTablePartitioned(hiveTable)) {
			List<ColumnStatisticsObj> columnStatisticsObjs = client.getTableColumnStatistics(
					hiveTable.getDbName(), hiveTable.getTableName(), getFieldNames(hiveTable.getSd().getCols()));
			return new CatalogColumnStatistics(HiveStatsUtil.createCatalogColumnStats(columnStatisticsObjs));
		} else {
			// TableColumnStats of partitioned table is unknown, the behavior is same as HIVE
			return CatalogColumnStatistics.UNKNOWN;
		}
	} catch (TException e) {
		throw new CatalogException(String.format("Failed to get table column stats of table %s",
												tablePath.getFullName()), e);
	}
}

Example #9

Source File: CatalogStatisticsTest.java From flink with Apache License 2.0

6 votes

private void createPartitionColumnStats(String part1, int part2, boolean unknown) throws Exception {
	ObjectPath path = ObjectPath.fromString("default_database.PartT");
	LinkedHashMap<String, String> partSpecMap = new LinkedHashMap<>();
	partSpecMap.put("part1", part1);
	partSpecMap.put("part2", String.valueOf(part2));
	CatalogPartitionSpec partSpec = new CatalogPartitionSpec(partSpecMap);

	CatalogColumnStatisticsDataLong longColStats = new CatalogColumnStatisticsDataLong(
			-123L, 763322L, 23L, 77L);
	CatalogColumnStatisticsDataString stringColStats = new CatalogColumnStatisticsDataString(
			152L, 43.5D, 20L, 0L);
	Map<String, CatalogColumnStatisticsDataBase> colStatsMap = new HashMap<>();
	colStatsMap.put("id", unknown ?
			new CatalogColumnStatisticsDataLong(null, null, null, null) :
			longColStats);
	colStatsMap.put("name", unknown ?
			new CatalogColumnStatisticsDataString(null, null, null, null) :
			stringColStats);
	catalog.alterPartitionColumnStatistics(
			path,
			partSpec,
			new CatalogColumnStatistics(colStatsMap),
			true);
}

Example #10

Source File: CatalogStatisticsTest.java From flink with Apache License 2.0

6 votes

private CatalogColumnStatistics createColumnStats() {
	CatalogColumnStatisticsDataBoolean booleanColStats = new CatalogColumnStatisticsDataBoolean(55L, 45L, 5L);
	CatalogColumnStatisticsDataLong longColStats = new CatalogColumnStatisticsDataLong(-123L, 763322L, 23L, 77L);
	CatalogColumnStatisticsDataString stringColStats = new CatalogColumnStatisticsDataString(152L, 43.5D, 20L, 0L);
	CatalogColumnStatisticsDataDate dateColStats =
			new CatalogColumnStatisticsDataDate(new Date(71L), new Date(17923L), 100L, 0L);
	CatalogColumnStatisticsDataDouble doubleColStats =
			new CatalogColumnStatisticsDataDouble(-123.35D, 7633.22D, 73L, 27L);
	Map<String, CatalogColumnStatisticsDataBase> colStatsMap = new HashMap<>(6);
	colStatsMap.put("b1", booleanColStats);
	colStatsMap.put("l2", longColStats);
	colStatsMap.put("s3", stringColStats);
	colStatsMap.put("d4", dateColStats);
	colStatsMap.put("dd5", doubleColStats);
	return new CatalogColumnStatistics(colStatsMap);
}

Example #11

Source File: HiveCatalogHiveMetadataTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAlterPartitionColumnStatistics() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable catalogTable = createPartitionedTable();
	catalog.createTable(path1, catalogTable, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path1, partitionSpec, createPartition(), true);
	Map<String, CatalogColumnStatisticsDataBase> columnStatisticsDataBaseMap = new HashMap<>();
	columnStatisticsDataBaseMap.put("first", new CatalogColumnStatisticsDataString(10L, 5.2, 3L, 100L));
	CatalogColumnStatistics catalogColumnStatistics = new CatalogColumnStatistics(columnStatisticsDataBaseMap);
	catalog.alterPartitionColumnStatistics(path1, partitionSpec, catalogColumnStatistics, false);

	checkEquals(catalogColumnStatistics, catalog.getPartitionColumnStatistics(path1, partitionSpec));
}

Example #12

Source File: HiveCatalogHiveMetadataTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAlterTableColumnStatistics() throws Exception {
	String hiveVersion = ((HiveCatalog) catalog).getHiveVersion();
	boolean supportDateStats = hiveVersion.compareTo(HiveShimLoader.HIVE_VERSION_V1_2_0) >= 0;
	catalog.createDatabase(db1, createDb(), false);
	TableSchema.Builder builder = TableSchema.builder()
			.field("first", DataTypes.STRING())
			.field("second", DataTypes.INT())
			.field("third", DataTypes.BOOLEAN())
			.field("fourth", DataTypes.DOUBLE())
			.field("fifth", DataTypes.BIGINT())
			.field("sixth", DataTypes.BYTES())
			.field("seventh", DataTypes.DECIMAL(10, 3))
			.field("eighth", DataTypes.DECIMAL(30, 3));
	if (supportDateStats) {
		builder.field("ninth", DataTypes.DATE());
	}
	TableSchema tableSchema = builder.build();
	CatalogTable catalogTable = new CatalogTableImpl(tableSchema, getBatchTableProperties(), TEST_COMMENT);
	catalog.createTable(path1, catalogTable, false);
	Map<String, CatalogColumnStatisticsDataBase> columnStatisticsDataBaseMap = new HashMap<>();
	columnStatisticsDataBaseMap.put("first", new CatalogColumnStatisticsDataString(10L, 5.2, 3L, 100L));
	columnStatisticsDataBaseMap.put("second", new CatalogColumnStatisticsDataLong(0L, 1000L, 3L, 0L));
	columnStatisticsDataBaseMap.put("third", new CatalogColumnStatisticsDataBoolean(15L, 20L, 3L));
	columnStatisticsDataBaseMap.put("fourth", new CatalogColumnStatisticsDataDouble(15.02, 20.01, 3L, 10L));
	columnStatisticsDataBaseMap.put("fifth", new CatalogColumnStatisticsDataLong(0L, 20L, 3L, 2L));
	columnStatisticsDataBaseMap.put("sixth", new CatalogColumnStatisticsDataBinary(150L, 20D, 3L));
	columnStatisticsDataBaseMap.put("seventh", new CatalogColumnStatisticsDataDouble(1.23, 99.456, 100L, 0L));
	columnStatisticsDataBaseMap.put("eighth", new CatalogColumnStatisticsDataDouble(0.123, 123456.789, 5723L, 19L));
	if (supportDateStats) {
		columnStatisticsDataBaseMap.put("ninth", new CatalogColumnStatisticsDataDate(
				new Date(71L), new Date(17923L), 132L, 0L));
	}
	CatalogColumnStatistics catalogColumnStatistics = new CatalogColumnStatistics(columnStatisticsDataBaseMap);
	catalog.alterTableColumnStatistics(path1, catalogColumnStatistics, false);

	checkEquals(catalogColumnStatistics, catalog.getTableColumnStatistics(path1));
}

Example #13

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException {
	checkNotNull(tablePath);

	if (!tableExists(tablePath)) {
		throw new TableNotExistException(getName(), tablePath);
	}

	CatalogColumnStatistics result = tableColumnStats.get(tablePath);
	return result != null ? result.copy() : CatalogColumnStatistics.UNKNOWN;
}

Example #14

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public CatalogColumnStatistics getPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (!partitionExists(tablePath, partitionSpec)) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}

	CatalogColumnStatistics result = partitionColumnStats.get(tablePath).get(partitionSpec);
	return result != null ? result.copy() : CatalogColumnStatistics.UNKNOWN;
}

Example #15

Source File: DatabaseCalciteSchema.java From flink with Apache License 2.0

5 votes

private Table convertConnectorTable(
		ConnectorCatalogTable<?, ?> table,
		ObjectPath tablePath) throws TableNotExistException {
	if (table.getTableSource().isPresent()) {
		TableSource<?> tableSource = table.getTableSource().get();
		if (!(tableSource instanceof StreamTableSource ||
				tableSource instanceof LookupableTableSource)) {
			throw new TableException(
					"Only StreamTableSource and LookupableTableSource can be used in Blink planner.");
		}
		if (!isStreamingMode && tableSource instanceof StreamTableSource &&
				!((StreamTableSource<?>) tableSource).isBounded()) {
			throw new TableException("Only bounded StreamTableSource can be used in batch mode.");
		}

		TableStats tableStats = TableStats.UNKNOWN;
		// TODO supports stats for partitionable table
		if (!table.isPartitioned()) {
			CatalogTableStatistics tableStatistics = catalog.getTableStatistics(tablePath);
			CatalogColumnStatistics columnStatistics = catalog.getTableColumnStatistics(tablePath);
			tableStats = convertToTableStats(tableStatistics, columnStatistics);
		}
		return new TableSourceTable<>(
				tableSource,
				isStreamingMode,
				FlinkStatistic.builder().tableStats(tableStats).build());
	} else {
		Optional<TableSinkTable> tableSinkTable = table.getTableSink()
			.map(tableSink -> new TableSinkTable<>(
				tableSink,
				FlinkStatistic.UNKNOWN()));
		if (tableSinkTable.isPresent()) {
			return tableSinkTable.get();
		} else {
			throw new TableException("Cannot convert a connector table " +
				"without either source or sink.");
		}
	}
}

Example #16

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public CatalogColumnStatistics getPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (!partitionExists(tablePath, partitionSpec)) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}

	CatalogColumnStatistics result = partitionColumnStats.get(tablePath).get(partitionSpec);
	return result != null ? result.copy() : CatalogColumnStatistics.UNKNOWN;
}

Example #17

Source File: GenericInMemoryCatalogTest.java From flink with Apache License 2.0

5 votes

@Test
public void testStatistics() throws Exception {
	// Table related
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable table = createTable();
	catalog.createTable(path1, table, false);

	CatalogTestUtil.checkEquals(catalog.getTableStatistics(path1), CatalogTableStatistics.UNKNOWN);
	CatalogTestUtil.checkEquals(catalog.getTableColumnStatistics(path1), CatalogColumnStatistics.UNKNOWN);

	CatalogTableStatistics tableStatistics = new CatalogTableStatistics(5, 2, 100, 575);
	catalog.alterTableStatistics(path1, tableStatistics, false);
	CatalogTestUtil.checkEquals(tableStatistics, catalog.getTableStatistics(path1));
	CatalogColumnStatistics columnStatistics = createColumnStats();
	catalog.alterTableColumnStatistics(path1, columnStatistics, false);
	CatalogTestUtil.checkEquals(columnStatistics, catalog.getTableColumnStatistics(path1));

	// Partition related
	catalog.createDatabase(db2, createDb(), false);
	CatalogTable table2 = createPartitionedTable();
	catalog.createTable(path2, table2, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path2, partitionSpec, createPartition(), false);

	CatalogTestUtil.checkEquals(catalog.getPartitionStatistics(path2, partitionSpec), CatalogTableStatistics.UNKNOWN);
	CatalogTestUtil.checkEquals(catalog.getPartitionColumnStatistics(path2, partitionSpec), CatalogColumnStatistics.UNKNOWN);

	catalog.alterPartitionStatistics(path2, partitionSpec, tableStatistics, false);
	CatalogTestUtil.checkEquals(tableStatistics, catalog.getPartitionStatistics(path2, partitionSpec));
	catalog.alterPartitionColumnStatistics(path2, partitionSpec, columnStatistics, false);
	CatalogTestUtil.checkEquals(columnStatistics, catalog.getPartitionColumnStatistics(path2, partitionSpec));

	// Clean up
	catalog.dropTable(path1, false);
	catalog.dropDatabase(db1, false);
	catalog.dropTable(path2, false);
	catalog.dropDatabase(db2, false);
}

Example #18

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public void alterPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogColumnStatistics columnStatistics,
		boolean ignoreIfNotExists) throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(columnStatistics);

	if (partitionExists(tablePath, partitionSpec)) {
		partitionColumnStats.get(tablePath).put(partitionSpec, columnStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}
}

Example #19

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public void alterTableColumnStatistics(ObjectPath tablePath, CatalogColumnStatistics columnStatistics,
		boolean ignoreIfNotExists) throws TableNotExistException {
	checkNotNull(tablePath);
	checkNotNull(columnStatistics);

	if (tableExists(tablePath)) {
		tableColumnStats.put(tablePath, columnStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new TableNotExistException(getName(), tablePath);
	}
}

Example #20

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public CatalogColumnStatistics getTableColumnStatistics(ObjectPath tablePath) throws TableNotExistException {
	checkNotNull(tablePath);

	if (!tableExists(tablePath)) {
		throw new TableNotExistException(getName(), tablePath);
	}

	CatalogColumnStatistics result = tableColumnStats.get(tablePath);
	return result != null ? result.copy() : CatalogColumnStatistics.UNKNOWN;
}

Example #21

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public void alterTableColumnStatistics(ObjectPath tablePath, CatalogColumnStatistics columnStatistics,
		boolean ignoreIfNotExists) throws TableNotExistException {
	checkNotNull(tablePath);
	checkNotNull(columnStatistics);

	if (tableExists(tablePath)) {
		tableColumnStats.put(tablePath, columnStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new TableNotExistException(getName(), tablePath);
	}
}

Example #22

Source File: GenericInMemoryCatalog.java From flink with Apache License 2.0

5 votes

@Override
public void alterPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogColumnStatistics columnStatistics,
		boolean ignoreIfNotExists) throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(columnStatistics);

	if (partitionExists(tablePath, partitionSpec)) {
		partitionColumnStats.get(tablePath).put(partitionSpec, columnStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}
}

Example #23

Source File: GenericInMemoryCatalogTest.java From flink with Apache License 2.0

5 votes

@Test
public void testStatistics() throws Exception {
	// Table related
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable table = createTable();
	catalog.createTable(path1, table, false);

	CatalogTestUtil.checkEquals(catalog.getTableStatistics(path1), CatalogTableStatistics.UNKNOWN);
	CatalogTestUtil.checkEquals(catalog.getTableColumnStatistics(path1), CatalogColumnStatistics.UNKNOWN);

	CatalogTableStatistics tableStatistics = new CatalogTableStatistics(5, 2, 100, 575);
	catalog.alterTableStatistics(path1, tableStatistics, false);
	CatalogTestUtil.checkEquals(tableStatistics, catalog.getTableStatistics(path1));
	CatalogColumnStatistics columnStatistics = createColumnStats();
	catalog.alterTableColumnStatistics(path1, columnStatistics, false);
	CatalogTestUtil.checkEquals(columnStatistics, catalog.getTableColumnStatistics(path1));

	// Partition related
	catalog.createDatabase(db2, createDb(), false);
	CatalogTable table2 = createPartitionedTable();
	catalog.createTable(path2, table2, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path2, partitionSpec, createPartition(), false);

	CatalogTestUtil.checkEquals(catalog.getPartitionStatistics(path2, partitionSpec), CatalogTableStatistics.UNKNOWN);
	CatalogTestUtil.checkEquals(catalog.getPartitionColumnStatistics(path2, partitionSpec), CatalogColumnStatistics.UNKNOWN);

	catalog.alterPartitionStatistics(path2, partitionSpec, tableStatistics, false);
	CatalogTestUtil.checkEquals(tableStatistics, catalog.getPartitionStatistics(path2, partitionSpec));
	catalog.alterPartitionColumnStatistics(path2, partitionSpec, columnStatistics, false);
	CatalogTestUtil.checkEquals(columnStatistics, catalog.getPartitionColumnStatistics(path2, partitionSpec));

	// Clean up
	catalog.dropTable(path1, false);
	catalog.dropDatabase(db1, false, false);
	catalog.dropTable(path2, false);
	catalog.dropDatabase(db2, false, false);
}

Example #24

Source File: HiveCatalogHiveMetadataTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAlterPartitionColumnStatistics() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable catalogTable = createPartitionedTable();
	catalog.createTable(path1, catalogTable, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path1, partitionSpec, createPartition(), true);
	Map<String, CatalogColumnStatisticsDataBase> columnStatisticsDataBaseMap = new HashMap<>();
	columnStatisticsDataBaseMap.put("first", new CatalogColumnStatisticsDataString(10, 5.2, 3, 100));
	CatalogColumnStatistics catalogColumnStatistics = new CatalogColumnStatistics(columnStatisticsDataBaseMap);
	catalog.alterPartitionColumnStatistics(path1, partitionSpec, catalogColumnStatistics, false);

	checkEquals(catalogColumnStatistics, catalog.getPartitionColumnStatistics(path1, partitionSpec));
}

Example #25

Source File: HiveCatalogHiveMetadataTest.java From flink with Apache License 2.0

5 votes

@Test
public void testAlterTableColumnStatistics() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	TableSchema tableSchema = TableSchema.builder()
										.field("first", DataTypes.STRING())
										.field("second", DataTypes.INT())
										.field("third", DataTypes.BOOLEAN())
										.field("fourth", DataTypes.DATE())
										.field("fifth", DataTypes.DOUBLE())
										.field("sixth", DataTypes.BIGINT())
										.field("seventh", DataTypes.BYTES())
										.build();
	CatalogTable catalogTable = new CatalogTableImpl(tableSchema, getBatchTableProperties(), TEST_COMMENT);
	catalog.createTable(path1, catalogTable, false);
	Map<String, CatalogColumnStatisticsDataBase> columnStatisticsDataBaseMap = new HashMap<>();
	columnStatisticsDataBaseMap.put("first", new CatalogColumnStatisticsDataString(10, 5.2, 3, 100));
	columnStatisticsDataBaseMap.put("second", new CatalogColumnStatisticsDataLong(0, 1000, 3, 0));
	columnStatisticsDataBaseMap.put("third", new CatalogColumnStatisticsDataBoolean(15, 20, 3));
	columnStatisticsDataBaseMap.put("fourth", new CatalogColumnStatisticsDataDate(new Date(71L), new Date(17923L), 1321, 0L));
	columnStatisticsDataBaseMap.put("fifth", new CatalogColumnStatisticsDataDouble(15.02, 20.01, 3, 10));
	columnStatisticsDataBaseMap.put("sixth", new CatalogColumnStatisticsDataLong(0, 20, 3, 2));
	columnStatisticsDataBaseMap.put("seventh", new CatalogColumnStatisticsDataBinary(150, 20, 3));
	CatalogColumnStatistics catalogColumnStatistics = new CatalogColumnStatistics(columnStatisticsDataBaseMap);
	catalog.alterTableColumnStatistics(path1, catalogColumnStatistics, false);

	checkEquals(catalogColumnStatistics, catalog.getTableColumnStatistics(path1));
}

Example #26

Source File: KuduCatalog.java From bahir-flink with Apache License 2.0

4 votes

@Override
public CatalogColumnStatistics getPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec) throws CatalogException {
    return CatalogColumnStatistics.UNKNOWN;
}

Example #27

Source File: CatalogTest.java From flink with Apache License 2.0

4 votes

protected void checkEquals(CatalogColumnStatistics cs1, CatalogColumnStatistics cs2) {
	CatalogTestUtil.checkEquals(cs1, cs2);
}

Example #28

Source File: CatalogTestUtil.java From flink with Apache License 2.0

4 votes

static void checkEquals(CatalogColumnStatistics cs1, CatalogColumnStatistics cs2) {
	checkEquals(cs1.getColumnStatisticsData(), cs2.getColumnStatisticsData());
	assertEquals(cs1.getProperties(), cs2.getProperties());
}

Example #29

Source File: CatalogTableStats.java From flink with Apache License 2.0

4 votes

public CatalogTableStats(CatalogTableStatistics catalogTableStatistics, CatalogColumnStatistics catalogColumnStatistics) {
	this.catalogTableStatistics = catalogTableStatistics;
	this.catalogColumnStatistics = catalogColumnStatistics;
}

Example #30

Source File: AbstractReadOnlyCatalog.java From bahir-flink with Apache License 2.0

4 votes

@Override
public void alterPartitionColumnStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogColumnStatistics columnStatistics, boolean ignoreIfNotExists) throws PartitionNotExistException, CatalogException {
    throw UNSUPPORTED_ERR;
}