org.apache.flink.table.catalog.stats.CatalogTableStatistics Java Examples

private void checkStatistics(int inputStat, int expectStat) throws Exception {
	catalog.dropTable(path1, true);

	Map<String, String> properties = new HashMap<>();
	properties.put(CatalogConfig.IS_GENERIC, "false");
	properties.put(StatsSetupConst.ROW_COUNT, String.valueOf(inputStat));
	properties.put(StatsSetupConst.NUM_FILES, String.valueOf(inputStat));
	properties.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(inputStat));
	properties.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(inputStat));
	CatalogTable catalogTable = new CatalogTableImpl(
			TableSchema.builder().field("f0", DataTypes.INT()).build(),
			properties,
			"");
	catalog.createTable(path1, catalogTable, false);

	CatalogTableStatistics statistics = catalog.getTableStatistics(path1);
	assertEquals(expectStat, statistics.getRowCount());
	assertEquals(expectStat, statistics.getFileCount());
	assertEquals(expectStat, statistics.getRawDataSize());
	assertEquals(expectStat, statistics.getTotalSize());
}
 

private void createPartitionStats(
		String part1, int part2, long rowCount) throws Exception {
	ObjectPath path = ObjectPath.fromString("default_database.PartT");

	LinkedHashMap<String, String> partSpecMap = new LinkedHashMap<>();
	partSpecMap.put("part1", part1);
	partSpecMap.put("part2", String.valueOf(part2));
	CatalogPartitionSpec partSpec = new CatalogPartitionSpec(partSpecMap);
	catalog.createPartition(
			path,
			partSpec,
			new CatalogPartitionImpl(new HashMap<>(), ""),
			true);
	catalog.alterPartitionStatistics(
			path,
			partSpec,
			new CatalogTableStatistics(rowCount, 10, 1000L, 2000L),
			true);
}
 

/**
 * Determine if statistics is need to be updated, if it needs to be updated and updated its parameters.
 * @param statistics original ``hive table statistics.
 * @param parameters new catalog table statistics parameters.
 * @return needUpdateStatistics flag which indicates whether need to update stats.
 */
private static boolean compareAndUpdateStatisticsProperties(CatalogTableStatistics statistics, Map<String, String> parameters) {
	boolean needUpdateStatistics;
	String oldRowCount = parameters.getOrDefault(StatsSetupConst.ROW_COUNT, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST);
	String oldTotalSize = parameters.getOrDefault(StatsSetupConst.TOTAL_SIZE, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST);
	String oldNumFiles = parameters.getOrDefault(StatsSetupConst.NUM_FILES, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST);
	String oldRawDataSize = parameters.getOrDefault(StatsSetupConst.RAW_DATA_SIZE, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST);
	needUpdateStatistics = statistics.getRowCount() != Long.parseLong(oldRowCount) || statistics.getTotalSize() != Long.parseLong(oldTotalSize)
		|| statistics.getFileCount() != Integer.parseInt(oldNumFiles) || statistics.getRawDataSize() != Long.parseLong(oldRawDataSize);
	if (needUpdateStatistics) {
		parameters.put(StatsSetupConst.ROW_COUNT, String.valueOf(statistics.getRowCount()));
		parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(statistics.getTotalSize()));
		parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(statistics.getFileCount()));
		parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(statistics.getRawDataSize()));
	}
	return needUpdateStatistics;
}
 

public static TableStats convertToTableStats(
		CatalogTableStatistics tableStatistics,
		CatalogColumnStatistics columnStatistics) {
	if (tableStatistics == null || tableStatistics.equals(CatalogTableStatistics.UNKNOWN)) {
		return TableStats.UNKNOWN;
	}

	long rowCount = tableStatistics.getRowCount();
	Map<String, ColumnStats> columnStatsMap = null;
	if (columnStatistics != null && !columnStatistics.equals(CatalogColumnStatistics.UNKNOWN)) {
		columnStatsMap = convertToColumnStatsMap(columnStatistics.getColumnStatisticsData());
	}
	if (columnStatsMap == null) {
		columnStatsMap = new HashMap<>();
	}
	return new TableStats(rowCount, columnStatsMap);
}
 

@Test
public void testGetStatsFromCatalog() throws Exception {
	EnvironmentSettings settings = EnvironmentSettings.newInstance().useBlinkPlanner().inBatchMode().build();
	TableEnvironment tEnv = TableEnvironment.create(settings);
	tEnv.registerTableSource("T1", new TestTableSource(true, tableSchema));
	tEnv.registerTableSource("T2", new TestTableSource(true, tableSchema));
	Catalog catalog = tEnv.getCatalog(tEnv.getCurrentCatalog()).orElse(null);
	assertNotNull(catalog);

	catalog.alterTableStatistics(ObjectPath.fromString("default_database.T1"),
			new CatalogTableStatistics(100, 10, 1000L, 2000L), true);
	catalog.alterTableStatistics(ObjectPath.fromString("default_database.T2"),
			new CatalogTableStatistics(100000000, 1000, 1000000000L, 2000000000L), true);
	catalog.alterTableColumnStatistics(ObjectPath.fromString("default_database.T1"), createColumnStats(), true);
	catalog.alterTableColumnStatistics(ObjectPath.fromString("default_database.T2"), createColumnStats(), true);

	Table table = tEnv.sqlQuery("select * from T1, T2 where T1.s3 = T2.s3");
	String result = tEnv.explain(table);
	// T1 is broadcast side
	String expected = TableTestUtil.readFromResource("/explain/testGetStatsFromCatalog.out");
	assertEquals(expected, TableTestUtil.replaceStageId(result));
}
 

public static TableStats convertToTableStats(
		CatalogTableStatistics tableStatistics,
		CatalogColumnStatistics columnStatistics) {
	long rowCount;
	if (tableStatistics != null && tableStatistics.getRowCount() >= 0) {
		rowCount = tableStatistics.getRowCount();
	} else {
		rowCount = TableStats.UNKNOWN.getRowCount();
	}

	Map<String, ColumnStats> columnStatsMap;
	if (columnStatistics != null) {
		columnStatsMap = convertToColumnStatsMap(columnStatistics.getColumnStatisticsData());
	} else {
		columnStatsMap = new HashMap<>();
	}
	return new TableStats(rowCount, columnStatsMap);
}
 

private static TableStats extractTableStats(
		Catalog catalog,
		ObjectIdentifier objectIdentifier) {
	final ObjectPath tablePath = objectIdentifier.toObjectPath();
	try {
		CatalogTableStatistics tableStatistics = catalog.getTableStatistics(tablePath);
		CatalogColumnStatistics columnStatistics = catalog.getTableColumnStatistics(tablePath);
		return convertToTableStats(tableStatistics, columnStatistics);
	} catch (TableNotExistException e) {
		throw new ValidationException(format(
			"Could not get statistic for table: [%s, %s, %s]",
			objectIdentifier.getCatalogName(),
			tablePath.getDatabaseName(),
			tablePath.getObjectName()), e);
	}
}
 

/**
 * Determine if statistics need to be updated or not.
 * @param newTableStats   new catalog table statistics.
 * @param parameters      original hive table statistics parameters.
 * @return                whether need to update stats.
 */
private boolean statsChanged(CatalogTableStatistics newTableStats, Map<String, String> parameters) {
	return newTableStats.getRowCount() != parsePositiveLongStat(parameters, StatsSetupConst.ROW_COUNT)
			|| newTableStats.getTotalSize() != parsePositiveLongStat(parameters, StatsSetupConst.TOTAL_SIZE)
			|| newTableStats.getFileCount() != parsePositiveIntStat(parameters, StatsSetupConst.NUM_FILES)
			|| newTableStats.getRawDataSize() != parsePositiveLongStat(parameters, StatsSetupConst.NUM_FILES);
}
 

@Test
public void testStatistics() throws Exception {
	// Table related
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable table = createTable();
	catalog.createTable(path1, table, false);

	CatalogTestUtil.checkEquals(catalog.getTableStatistics(path1), CatalogTableStatistics.UNKNOWN);
	CatalogTestUtil.checkEquals(catalog.getTableColumnStatistics(path1), CatalogColumnStatistics.UNKNOWN);

	CatalogTableStatistics tableStatistics = new CatalogTableStatistics(5, 2, 100, 575);
	catalog.alterTableStatistics(path1, tableStatistics, false);
	CatalogTestUtil.checkEquals(tableStatistics, catalog.getTableStatistics(path1));
	CatalogColumnStatistics columnStatistics = createColumnStats();
	catalog.alterTableColumnStatistics(path1, columnStatistics, false);
	CatalogTestUtil.checkEquals(columnStatistics, catalog.getTableColumnStatistics(path1));

	// Partition related
	catalog.createDatabase(db2, createDb(), false);
	CatalogTable table2 = createPartitionedTable();
	catalog.createTable(path2, table2, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path2, partitionSpec, createPartition(), false);

	CatalogTestUtil.checkEquals(catalog.getPartitionStatistics(path2, partitionSpec), CatalogTableStatistics.UNKNOWN);
	CatalogTestUtil.checkEquals(catalog.getPartitionColumnStatistics(path2, partitionSpec), CatalogColumnStatistics.UNKNOWN);

	catalog.alterPartitionStatistics(path2, partitionSpec, tableStatistics, false);
	CatalogTestUtil.checkEquals(tableStatistics, catalog.getPartitionStatistics(path2, partitionSpec));
	catalog.alterPartitionColumnStatistics(path2, partitionSpec, columnStatistics, false);
	CatalogTestUtil.checkEquals(columnStatistics, catalog.getPartitionColumnStatistics(path2, partitionSpec));

	// Clean up
	catalog.dropTable(path1, false);
	catalog.dropDatabase(db1, false);
	catalog.dropTable(path2, false);
	catalog.dropDatabase(db2, false);
}
 

private Table convertConnectorTable(
		ConnectorCatalogTable<?, ?> table,
		ObjectPath tablePath) throws TableNotExistException {
	if (table.getTableSource().isPresent()) {
		TableSource<?> tableSource = table.getTableSource().get();
		if (!(tableSource instanceof StreamTableSource ||
				tableSource instanceof LookupableTableSource)) {
			throw new TableException(
					"Only StreamTableSource and LookupableTableSource can be used in Blink planner.");
		}
		if (!isStreamingMode && tableSource instanceof StreamTableSource &&
				!((StreamTableSource<?>) tableSource).isBounded()) {
			throw new TableException("Only bounded StreamTableSource can be used in batch mode.");
		}

		TableStats tableStats = TableStats.UNKNOWN;
		// TODO supports stats for partitionable table
		if (!table.isPartitioned()) {
			CatalogTableStatistics tableStatistics = catalog.getTableStatistics(tablePath);
			CatalogColumnStatistics columnStatistics = catalog.getTableColumnStatistics(tablePath);
			tableStats = convertToTableStats(tableStatistics, columnStatistics);
		}
		return new TableSourceTable<>(
				tableSource,
				isStreamingMode,
				FlinkStatistic.builder().tableStats(tableStats).build());
	} else {
		Optional<TableSinkTable> tableSinkTable = table.getTableSink()
			.map(tableSink -> new TableSinkTable<>(
				tableSink,
				FlinkStatistic.UNKNOWN()));
		if (tableSinkTable.isPresent()) {
			return tableSinkTable.get();
		} else {
			throw new TableException("Cannot convert a connector table " +
				"without either source or sink.");
		}
	}
}
 

@Override
public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (!partitionExists(tablePath, partitionSpec)) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}

	CatalogTableStatistics result = partitionStats.get(tablePath).get(partitionSpec);
	return result != null ? result.copy() : CatalogTableStatistics.UNKNOWN;
}
 

@Override
public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException {
	checkNotNull(tablePath);

	if (!tableExists(tablePath)) {
		throw new TableNotExistException(getName(), tablePath);
	}
	if (!isPartitionedTable(tablePath)) {
		CatalogTableStatistics result = tableStats.get(tablePath);
		return result != null ? result.copy() : CatalogTableStatistics.UNKNOWN;
	} else {
		return CatalogTableStatistics.UNKNOWN;
	}
}
 

@Test
public void testAlterPartitionTableStats() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable catalogTable = createPartitionedTable();
	catalog.createTable(path1, catalogTable, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path1, partitionSpec, createPartition(), true);
	CatalogTableStatistics stats = new CatalogTableStatistics(100, 1, 1000, 10000);
	catalog.alterPartitionStatistics(path1, partitionSpec, stats, false);
	CatalogTableStatistics actual = catalog.getPartitionStatistics(path1, partitionSpec);
	assertEquals(stats.getRowCount(), actual.getRowCount());
	assertEquals(stats.getRawDataSize(), actual.getRawDataSize());
}
 

@Test
public void testAlterTableStats_partitionedTable() throws Exception {
	// alterTableStats() should do nothing for partitioned tables
	// getTableStats() should return unknown column stats for partitioned tables
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable catalogTable = createPartitionedTable();
	catalog.createTable(path1, catalogTable, false);

	CatalogTableStatistics stats = new CatalogTableStatistics(100, 1, 1000, 10000);

	catalog.alterTableStatistics(path1, stats, false);

	assertEquals(CatalogTableStatistics.UNKNOWN, catalog.getTableStatistics(path1));
}
 

@Test
public void testAlterTableStats() throws Exception{
	// Non-partitioned table
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable table = createTable();
	catalog.createTable(path1, table, false);
	CatalogTableStatistics tableStats = new CatalogTableStatistics(100, 10, 1000, 10000);
	catalog.alterTableStatistics(path1, tableStats, false);
	CatalogTableStatistics actual = catalog.getTableStatistics(path1);

	// we don't check fileCount and totalSize here for hive will automatically calc and set to real num.
	assertEquals(tableStats.getRowCount(), actual.getRowCount());
	assertEquals(tableStats.getRawDataSize(), actual.getRawDataSize());
}
 

@Test
public void testGetPartitionStats() throws Exception{
	catalog.createDatabase(db1, createDb(), false);
	catalog.createTable(path1, createPartitionedTable(), false);
	catalog.createPartition(path1, createPartitionSpec(), createPartition(), false);
	CatalogTableStatistics tableStatistics = catalog.getPartitionStatistics(path1, createPartitionSpec());
	assertEquals(-1, tableStatistics.getFileCount());
	assertEquals(-1, tableStatistics.getRawDataSize());
	assertEquals(-1, tableStatistics.getTotalSize());
	assertEquals(-1, tableStatistics.getRowCount());
}
 

@Override
public void alterTableStatistics(ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
		throws TableNotExistException {
	checkNotNull(tablePath);
	checkNotNull(tableStatistics);

	if (tableExists(tablePath)) {
		tableStats.put(tablePath, tableStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new TableNotExistException(getName(), tablePath);
	}
}
 

/**
 * Update original table statistics parameters.
 * @param newTableStats   new catalog table statistics.
 * @param parameters      original hive table statistics parameters.
 */
private void updateStats(CatalogTableStatistics newTableStats, Map<String, String> parameters) {
	parameters.put(StatsSetupConst.ROW_COUNT, String.valueOf(newTableStats.getRowCount()));
	parameters.put(StatsSetupConst.TOTAL_SIZE, String.valueOf(newTableStats.getTotalSize()));
	parameters.put(StatsSetupConst.NUM_FILES, String.valueOf(newTableStats.getFileCount()));
	parameters.put(StatsSetupConst.RAW_DATA_SIZE, String.valueOf(newTableStats.getRawDataSize()));
}
 

private static CatalogTableStatistics createCatalogTableStatistics(Map<String, String> parameters) {
	return new CatalogTableStatistics(
			parsePositiveLongStat(parameters, StatsSetupConst.ROW_COUNT),
			parsePositiveIntStat(parameters, StatsSetupConst.NUM_FILES),
			parsePositiveLongStat(parameters, StatsSetupConst.TOTAL_SIZE),
			parsePositiveLongStat(parameters, StatsSetupConst.RAW_DATA_SIZE));
}
 

@Override
public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException,
		CatalogException {
	Table hiveTable = getHiveTable(tablePath);
	if (!isTablePartitioned(hiveTable)) {
		return createCatalogTableStatistics(hiveTable.getParameters());
	} else {
		return CatalogTableStatistics.UNKNOWN;
	}
}
 

static void checkEquals(CatalogTableStatistics ts1, CatalogTableStatistics ts2) {
	assertEquals(ts1.getRowCount(), ts2.getRowCount());
	assertEquals(ts1.getFileCount(), ts2.getFileCount());
	assertEquals(ts1.getTotalSize(), ts2.getTotalSize());
	assertEquals(ts1.getRawDataSize(), ts2.getRawDataSize());
	assertEquals(ts1.getProperties(), ts2.getProperties());
}
 

@Override
public void alterPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogTableStatistics partitionStatistics,
		boolean ignoreIfNotExists) throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partitionStatistics);

	if (partitionExists(tablePath, partitionSpec)) {
		partitionStats.get(tablePath).put(partitionSpec, partitionStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}
}
 

@Override
public CatalogTableStatistics getPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec)
		throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);

	if (!partitionExists(tablePath, partitionSpec)) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}

	CatalogTableStatistics result = partitionStats.get(tablePath).get(partitionSpec);
	return result != null ? result.copy() : CatalogTableStatistics.UNKNOWN;
}
 

@Override
public CatalogTableStatistics getTableStatistics(ObjectPath tablePath) throws TableNotExistException {
	checkNotNull(tablePath);

	if (!tableExists(tablePath)) {
		throw new TableNotExistException(getName(), tablePath);
	}
	if (!isPartitionedTable(tablePath)) {
		CatalogTableStatistics result = tableStats.get(tablePath);
		return result != null ? result.copy() : CatalogTableStatistics.UNKNOWN;
	} else {
		return CatalogTableStatistics.UNKNOWN;
	}
}
 

@Override
public void alterTableStatistics(ObjectPath tablePath, CatalogTableStatistics tableStatistics, boolean ignoreIfNotExists)
		throws TableNotExistException {
	checkNotNull(tablePath);
	checkNotNull(tableStatistics);

	if (tableExists(tablePath)) {
		tableStats.put(tablePath, tableStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new TableNotExistException(getName(), tablePath);
	}
}
 

@Override
public void alterPartitionStatistics(ObjectPath tablePath, CatalogPartitionSpec partitionSpec, CatalogTableStatistics partitionStatistics,
		boolean ignoreIfNotExists) throws PartitionNotExistException {
	checkNotNull(tablePath);
	checkNotNull(partitionSpec);
	checkNotNull(partitionStatistics);

	if (partitionExists(tablePath, partitionSpec)) {
		partitionStats.get(tablePath).put(partitionSpec, partitionStatistics.copy());
	} else if (!ignoreIfNotExists) {
		throw new PartitionNotExistException(getName(), tablePath, partitionSpec);
	}
}
 

@Test
public void testAlterPartitionTableStats() throws Exception {
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable catalogTable = createPartitionedTable();
	catalog.createTable(path1, catalogTable, false);
	CatalogPartitionSpec partitionSpec = createPartitionSpec();
	catalog.createPartition(path1, partitionSpec, createPartition(), true);
	CatalogTableStatistics stats = new CatalogTableStatistics(100, 1, 1000, 10000);
	catalog.alterPartitionStatistics(path1, partitionSpec, stats, false);
	CatalogTableStatistics actual = catalog.getPartitionStatistics(path1, partitionSpec);
	assertEquals(stats.getRowCount(), actual.getRowCount());
	assertEquals(stats.getRawDataSize(), actual.getRawDataSize());
}
 

private void alterTableStatisticsWithUnknownRowCount(
		Catalog catalog,
		String tableName) throws TableNotExistException, TablePartitionedException {
	catalog.alterTableStatistics(new ObjectPath(databaseName, tableName),
			new CatalogTableStatistics(CatalogTableStatistics.UNKNOWN.getRowCount(), 1, 10000, 200000), true);
	catalog.alterTableColumnStatistics(new ObjectPath(databaseName, tableName), createColumnStats(), true);
}
 

private static CatalogTableStatistics createCatalogTableStatistics(Map<String, String> parameters) {
	long rowRount = Long.parseLong(parameters.getOrDefault(StatsSetupConst.ROW_COUNT, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST));
	long totalSize = Long.parseLong(parameters.getOrDefault(StatsSetupConst.TOTAL_SIZE, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST));
	int numFiles = Integer.parseInt(parameters.getOrDefault(StatsSetupConst.NUM_FILES, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST));
	long rawDataSize = Long.parseLong(parameters.getOrDefault(StatsSetupConst.RAW_DATA_SIZE, HiveStatsUtil.DEFAULT_STATS_ZERO_CONST));
	return new CatalogTableStatistics(rowRount, numFiles, totalSize, rawDataSize);
}
 

@Test
public void testAlterTableStats_partitionedTable() throws Exception {
	// alterTableStats() should do nothing for partitioned tables
	// getTableStats() should return unknown column stats for partitioned tables
	catalog.createDatabase(db1, createDb(), false);
	CatalogTable catalogTable = createPartitionedTable();
	catalog.createTable(path1, catalogTable, false);

	CatalogTableStatistics stats = new CatalogTableStatistics(100, 1, 1000, 10000);

	catalog.alterTableStatistics(path1, stats, false);

	assertEquals(CatalogTableStatistics.UNKNOWN, catalog.getTableStatistics(path1));
}