org.apache.hadoop.hive.metastore.api.ColumnStatistics Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.ColumnStatistics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void transformInboundSetPartitionStatsRequest() throws Exception {
  SetPartitionsStatsRequest setPartitionsStatsRequest = new SetPartitionsStatsRequest();
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  setPartitionsStatsRequest.setColStats(Lists.newArrayList(columnStatistics));
  SetPartitionsStatsRequest result = databaseMapping
      .transformInboundSetPartitionStatsRequest(setPartitionsStatsRequest);
  assertThat(result, is(sameInstance(setPartitionsStatsRequest)));
  ColumnStatistics resultColStats = result.getColStats().get(0);
  assertThat(resultColStats, is(sameInstance(columnStatistics)));
  assertThat(resultColStats.getStatsDesc(), is(sameInstance(statsDesc)));
  assertThat(resultColStats.getStatsDesc().getDbName(), is(IN_DB_NAME));
}
 
Example #2
Source File: HiveStatsUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ColumnStatistics createHiveColumnStatistics(
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		StorageDescriptor sd,
		ColumnStatisticsDesc desc,
		String hiveVersion) {
	List<ColumnStatisticsObj> colStatsList = new ArrayList<>();

	for (FieldSchema field : sd.getCols()) {
		String hiveColName = field.getName();
		String hiveColType = field.getType();
		CatalogColumnStatisticsDataBase flinkColStat = colStats.get(field.getName());
		if (null != flinkColStat) {
			ColumnStatisticsData statsData = getColumnStatisticsData(
					HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(hiveColType)),
					flinkColStat,
					hiveVersion);
			ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj(hiveColName, hiveColType, statsData);
			colStatsList.add(columnStatisticsObj);
		}
	}

	return new ColumnStatistics(desc, colStatsList);
}
 
Example #3
Source File: HiveStatsUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ColumnStatistics createHiveColumnStatistics(
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		StorageDescriptor sd,
		ColumnStatisticsDesc desc) {
	List<ColumnStatisticsObj> colStatsList = new ArrayList<>();

	for (FieldSchema field : sd.getCols()) {
		String hiveColName = field.getName();
		String hiveColType = field.getType();
		CatalogColumnStatisticsDataBase flinkColStat = colStats.get(field.getName());
		if (null != flinkColStat) {
			ColumnStatisticsData statsData =
					getColumnStatisticsData(HiveTypeUtil.toFlinkType(TypeInfoUtils.getTypeInfoFromTypeString(hiveColType)), flinkColStat);
			ColumnStatisticsObj columnStatisticsObj = new ColumnStatisticsObj(hiveColName, hiveColType, statsData);
			colStatsList.add(columnStatisticsObj);
		}
	}

	return new ColumnStatistics(desc, colStatsList);
}
 
Example #4
Source File: PartitionedTableMetadataUpdateReplication.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private PartitionsAndStatistics filterOnReplicatedPartitions(
    CloseableMetaStoreClient replicaClient,
    PartitionsAndStatistics sourcePartitionsAndStatistics,
    List<FieldSchema> partitionKeys)
  throws TException {
  Map<Partition, ColumnStatistics> statisticsByPartition = new LinkedHashMap<>();
  for (Partition partition : sourcePartitionsAndStatistics.getPartitions()) {
    try {
      replicaClient.getPartition(replicaDatabaseName, replicaTableName, partition.getValues());
      statisticsByPartition.put(partition, sourcePartitionsAndStatistics.getStatisticsForPartition(partition));
    } catch (NoSuchObjectException e) {
      LOG.debug("Partition {} doesn't exist, skipping it...", Warehouse.getQualifiedName(partition));
    }
  }
  return new PartitionsAndStatistics(partitionKeys, statisticsByPartition);
}
 
Example #5
Source File: PartitionsAndStatisticsTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"), newFieldSchema("c"));
  Table table = newTable("t1", "db1", partitionKeys, newStorageDescriptor(new File("bla"), "col1"));
  List<Partition> partitions = Lists.newArrayList(newPartition(table, "b", "d"));
  statisticsPerPartitionName.put("a=b/c=d", columnStats);

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);
  List<String> expectedName = Lists.newArrayList("a=b/c=d");

  assertThat(partitionsAndStatistics.getPartitionNames(), is(expectedName));
  assertThat(partitionsAndStatistics.getPartitions(), is(partitions));
  assertThat(partitionsAndStatistics.getPartitionKeys(), is(partitionKeys));
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db1", "t1");
  statsDesc.setPartName("a=b/c=d");
  ColumnStatistics expectedStats = new ColumnStatistics(statsDesc, columnStats);
  assertThat(partitionsAndStatistics.getStatisticsForPartition(partitions.get(0)), is(expectedStats));
}
 
Example #6
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean update_table_column_statistics(ColumnStatistics stats_obj)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(stats_obj.getStatsDesc().getDbName());
  return mapping.getClient().update_table_column_statistics(mapping.transformInboundColumnStatistics(stats_obj));
}
 
Example #7
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void transformOutboundColumnStatistics() throws Exception {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  ColumnStatistics result = databaseMapping.transformOutboundColumnStatistics(columnStatistics);
  assertThat(result, is(sameInstance(columnStatistics)));
  assertThat(result.getStatsDesc(), is(sameInstance(columnStatistics.getStatsDesc())));
  assertThat(result.getStatsDesc().getDbName(), is(OUT_DB_NAME));
}
 
Example #8
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void transformInboundColumnStatistics() throws Exception {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
  statsDesc.setDbName(DB_NAME);
  columnStatistics.setStatsDesc(statsDesc);
  ColumnStatistics result = databaseMapping.transformInboundColumnStatistics(columnStatistics);
  assertThat(result, is(sameInstance(columnStatistics)));
  assertThat(result.getStatsDesc(), is(sameInstance(columnStatistics.getStatsDesc())));
  assertThat(result.getStatsDesc().getDbName(), is(IN_DB_NAME));
}
 
Example #9
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean set_aggr_stats_for(SetPartitionsStatsRequest request)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  if (!request.getColStats().isEmpty()) {
    DatabaseMapping mapping = databaseMappingService
        .databaseMapping(request.getColStats().get(0).getStatsDesc().getDbName());
    for (ColumnStatistics stats : request.getColStats()) {
      mapping.checkWritePermissions(stats.getStatsDesc().getDbName());
    }
    return mapping.getClient().set_aggr_stats_for(mapping.transformInboundSetPartitionStatsRequest(request));
  }
  return false;
}
 
Example #10
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_partition_column_statistics(
    String db_name,
    String tbl_name,
    String part_name,
    String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_partition_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, part_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 
Example #11
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_table_column_statistics(String db_name, String tbl_name, String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_table_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 
Example #12
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean update_partition_column_statistics(ColumnStatistics stats_obj)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(stats_obj.getStatsDesc().getDbName());
  return mapping.getClient().update_partition_column_statistics(mapping.transformInboundColumnStatistics(stats_obj));
}
 
Example #13
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void update_partition_column_statistics() throws TException {
  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(true, DB_P, "table");
  ColumnStatistics columnStatistics = new ColumnStatistics(columnStatisticsDesc, Collections.emptyList());
  ColumnStatistics inboundColumnStatistics = new ColumnStatistics();
  when(primaryMapping.transformInboundColumnStatistics(columnStatistics)).thenReturn(inboundColumnStatistics);
  when(primaryClient.update_partition_column_statistics(inboundColumnStatistics)).thenReturn(true);
  boolean result = handler.update_partition_column_statistics(columnStatistics);
  verify(primaryMapping).checkWritePermissions(DB_P);
  assertThat(result, is(true));
}
 
Example #14
Source File: DatabaseMappingImpl.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public SetPartitionsStatsRequest transformInboundSetPartitionStatsRequest(SetPartitionsStatsRequest request) {
  if (request.isSetColStats()) {
    for (ColumnStatistics stats : request.getColStats()) {
      transformInboundColumnStatistics(stats);
    }
  }
  return request;
}
 
Example #15
Source File: DatabaseMappingImpl.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public ColumnStatistics transformOutboundColumnStatistics(ColumnStatistics columnStatistics) {
  columnStatistics
      .getStatsDesc()
      .setDbName(metaStoreMapping.transformOutboundDatabaseName(columnStatistics.getStatsDesc().getDbName()));
  return columnStatistics;
}
 
Example #16
Source File: ReplicaTableFactoryTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Override
public ColumnStatistics transform(ColumnStatistics columnStatistics) {
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "new_db", "new_table");
  statsDesc.setPartName("part=newPart");
  columnStatistics.setStatsDesc(statsDesc);
  return columnStatistics;
}
 
Example #17
Source File: ReplicaTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void alteringExistingUnpartitionedReplicaTableWithNoStatsSucceeds() throws Exception {
  tableAndStatistics = new TableAndStatistics(sourceTable, null);
  existingReplicaTable.getParameters().put(REPLICATION_EVENT.parameterName(), "previousEventId");
  replica.updateMetadata(EVENT_ID, tableAndStatistics, DB_NAME, TABLE_NAME, mockReplicaLocationManager);
  verify(alterTableService).alterTable(eq(mockMetaStoreClient), eq(existingReplicaTable), any(Table.class));
  verify(mockMetaStoreClient, never()).updateTableColumnStatistics(any(ColumnStatistics.class));
  verify(mockReplicaLocationManager, never()).addCleanUpLocation(anyString(), any(Path.class));
}
 
Example #18
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void update_table_column_statistics() throws TException {
  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(true, DB_P, "table");
  ColumnStatistics columnStatistics = new ColumnStatistics(columnStatisticsDesc, Collections.emptyList());
  ColumnStatistics inboundColumnStatistics = new ColumnStatistics();
  when(primaryMapping.transformInboundColumnStatistics(columnStatistics)).thenReturn(inboundColumnStatistics);
  when(primaryClient.update_table_column_statistics(inboundColumnStatistics)).thenReturn(true);
  boolean result = handler.update_table_column_statistics(columnStatistics);
  verify(primaryMapping).checkWritePermissions(DB_P);
  assertThat(result, is(true));
}
 
Example #19
Source File: ThriftHiveMetastoreClient.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public void setPartitionColumnStatistics(String databaseName, String tableName, String partitionName, List<ColumnStatisticsObj> statistics)
        throws TException
{
    ColumnStatisticsDesc statisticsDescription = new ColumnStatisticsDesc(false, databaseName, tableName);
    statisticsDescription.setPartName(partitionName);
    ColumnStatistics request = new ColumnStatistics(statisticsDescription, statistics);
    client.update_partition_column_statistics(request);
}
 
Example #20
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void get_table_column_statistics() throws TException {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatistics outboundColumnStatistics = new ColumnStatistics();
  when(primaryClient.get_table_column_statistics(DB_P, "table", "columnName")).thenReturn(columnStatistics);
  when(primaryMapping.transformOutboundColumnStatistics(columnStatistics)).thenReturn(outboundColumnStatistics);
  ColumnStatistics result = handler.get_table_column_statistics(DB_P, "table", "columnName");
  assertThat(result, is(outboundColumnStatistics));
}
 
Example #21
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void get_partition_column_statistics() throws TException {
  ColumnStatistics columnStatistics = new ColumnStatistics();
  ColumnStatistics outboundColumnStatistics = new ColumnStatistics();
  when(primaryClient.get_partition_column_statistics(DB_P, "table", "partitionName", "columnName"))
      .thenReturn(columnStatistics);
  when(primaryMapping.transformOutboundColumnStatistics(columnStatistics)).thenReturn(outboundColumnStatistics);
  ColumnStatistics result = handler.get_partition_column_statistics(DB_P, "table", "partitionName", "columnName");
  assertThat(result, is(outboundColumnStatistics));
}
 
Example #22
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void set_aggr_stats_for() throws TException {
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(true, DB_P, "table");
  ColumnStatistics colStatistics = new ColumnStatistics(statsDesc, Collections.emptyList());
  List<ColumnStatistics> colStats = Collections.singletonList(colStatistics);
  SetPartitionsStatsRequest request = new SetPartitionsStatsRequest(colStats);
  SetPartitionsStatsRequest inboundRequest = new SetPartitionsStatsRequest();
  when(primaryMapping.transformInboundSetPartitionStatsRequest(request)).thenReturn(inboundRequest);
  when(primaryClient.set_aggr_stats_for(inboundRequest)).thenReturn(true);
  boolean result = handler.set_aggr_stats_for(request);
  assertThat(result, is(true));
}
 
Example #23
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public ColumnStatistics get_partition_column_statistics(
    final String dbName,
    final String tblName,
    final String partName,
    final String colName
) throws TException {
    throw unimplemented("get_partition_column_statistics", new Object[]{dbName, tblName, partName, colName});
}
 
Example #24
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public ColumnStatistics get_table_column_statistics(final String dbName, final String tblName,
                                                    final String colName)
    throws TException {
    throw unimplemented("get_table_column_statistics", new Object[]{dbName, tblName, colName});
}
 
Example #25
Source File: HiveStatsUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Create columnStatistics from the given Hive column stats of a hive table.
 */
public static ColumnStatistics createTableColumnStats(
		Table hiveTable,
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		String hiveVersion) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(true, hiveTable.getDbName(), hiveTable.getTableName());
	return createHiveColumnStatistics(colStats, hiveTable.getSd(), desc, hiveVersion);
}
 
Example #26
Source File: HiveStatsUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Create columnStatistics from the given Hive column stats of a hive partition.
 */
public static ColumnStatistics createPartitionColumnStats(
		Partition hivePartition,
		String partName,
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		String hiveVersion) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, hivePartition.getDbName(), hivePartition.getTableName());
	desc.setPartName(partName);
	return createHiveColumnStatistics(colStats, hivePartition.getSd(), desc, hiveVersion);
}
 
Example #27
Source File: AuthorizingObjectStore.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public ColumnStatistics getTableColumnStatistics(String dbName,
    String tableName, List<String> colNames) throws MetaException,
    NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tableName));
  }
  return super.getTableColumnStatistics(dbName, tableName, colNames);
}
 
Example #28
Source File: AuthorizingObjectStore.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public List<ColumnStatistics> getPartitionColumnStatistics(
    String dbName, String tblName, List<String> partNames,
    List<String> colNames) throws MetaException, NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tblName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tblName));
  }
  return super.getPartitionColumnStatistics(dbName, tblName, partNames,
      colNames);
}
 
Example #29
Source File: AuthorizingObjectStoreV2.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public ColumnStatistics getTableColumnStatistics(String dbName,
    String tableName, List<String> colNames) throws MetaException,
    NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tableName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tableName));
  }
  return super.getTableColumnStatistics(dbName, tableName, colNames);
}
 
Example #30
Source File: AuthorizingObjectStoreV2.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
@Override
public List<ColumnStatistics> getPartitionColumnStatistics(
    String dbName, String tblName, List<String> partNames,
    List<String> colNames) throws MetaException, NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tblName)).isEmpty()) {
    throw new MetaException(getNoAccessMessageForTable(dbName, tblName));
  }
  return super.getPartitionColumnStatistics(dbName, tblName, partNames,
      colNames);
}