Java Code Examples for org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc#setPartName()

The following examples show how to use org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc#setPartName() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void typical() throws Exception {
  List<FieldSchema> partitionKeys = Lists.newArrayList(newFieldSchema("a"), newFieldSchema("c"));
  Table table = newTable("t1", "db1", partitionKeys, newStorageDescriptor(new File("bla"), "col1"));
  List<Partition> partitions = Lists.newArrayList(newPartition(table, "b", "d"));
  statisticsPerPartitionName.put("a=b/c=d", columnStats);

  PartitionsAndStatistics partitionsAndStatistics = new PartitionsAndStatistics(partitionKeys, partitions,
      statisticsPerPartitionName);
  List<String> expectedName = Lists.newArrayList("a=b/c=d");

  assertThat(partitionsAndStatistics.getPartitionNames(), is(expectedName));
  assertThat(partitionsAndStatistics.getPartitions(), is(partitions));
  assertThat(partitionsAndStatistics.getPartitionKeys(), is(partitionKeys));
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "db1", "t1");
  statsDesc.setPartName("a=b/c=d");
  ColumnStatistics expectedStats = new ColumnStatistics(statsDesc, columnStats);
  assertThat(partitionsAndStatistics.getStatisticsForPartition(partitions.get(0)), is(expectedStats));
}
 
Example 2
@Override
public void setPartitionColumnStatistics(String databaseName, String tableName, String partitionName, List<ColumnStatisticsObj> statistics)
        throws TException
{
    ColumnStatisticsDesc statisticsDescription = new ColumnStatisticsDesc(false, databaseName, tableName);
    statisticsDescription.setPartName(partitionName);
    ColumnStatistics request = new ColumnStatistics(statisticsDescription, statistics);
    client.update_partition_column_statistics(request);
}
 
Example 3
Source Project: flink   File: HiveStatsUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create columnStatistics from the given Hive column stats of a hive partition.
 */
public static ColumnStatistics createPartitionColumnStats(
		Partition hivePartition,
		String partName,
		Map<String, CatalogColumnStatisticsDataBase> colStats) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, hivePartition.getDbName(), hivePartition.getTableName());
	desc.setPartName(partName);
	return createHiveColumnStatistics(colStats, hivePartition.getSd(), desc);
}
 
Example 4
ColumnStatistics newReplicaPartitionStatistics(
    Table replicaTable,
    Partition replicaPartition,
    ColumnStatistics sourcePartitionStatistics) {
  ColumnStatisticsDesc statisticsDesc = new ColumnStatisticsDesc(false, replicaPartition.getDbName(),
      replicaPartition.getTableName());
  try {
    statisticsDesc.setPartName(Warehouse.makePartName(replicaTable.getPartitionKeys(), replicaPartition.getValues()));
  } catch (MetaException e) {
    throw new RuntimeException(e);
  }

  return columnStatisticsTransformation
      .transform(new ColumnStatistics(statisticsDesc, sourcePartitionStatistics.getStatsObj()));
}
 
Example 5
Source Project: circus-train   File: ReplicaTest.java    License: Apache License 2.0 5 votes vote down vote up
private ColumnStatistics newPartitionStatistics(String... values) {
  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj(COLUMN_A, "string",
      new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(0, 1)));
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj(COLUMN_B, "string",
      new ColumnStatisticsData(_Fields.LONG_STATS, new LongColumnStatsData(1, 2)));
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  statsDesc.setPartName(partitionName(values));
  return new ColumnStatistics(statsDesc, columnStatisticsObjs);
}
 
Example 6
@Override
public ColumnStatistics transform(ColumnStatistics columnStatistics) {
  ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc(false, "new_db", "new_table");
  statsDesc.setPartName("part=newPart");
  columnStatistics.setStatsDesc(statsDesc);
  return columnStatistics;
}
 
Example 7
@Test
public void newReplicaPartitionStatistics() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 
Example 8
@Test
public void newReplicaPartitionStatisticsWithTransformation() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TableTransformation.IDENTITY,
      PartitionTransformation.IDENTITY, COLUMN_STATISTICS_TRANSFORMATION);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is("new_db"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is("new_table"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("part=newPart"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 
Example 9
Source Project: flink   File: HiveStatsUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create columnStatistics from the given Hive column stats of a hive partition.
 */
public static ColumnStatistics createPartitionColumnStats(
		Partition hivePartition,
		String partName,
		Map<String, CatalogColumnStatisticsDataBase> colStats,
		String hiveVersion) {
	ColumnStatisticsDesc desc = new ColumnStatisticsDesc(false, hivePartition.getDbName(), hivePartition.getTableName());
	desc.setPartName(partName);
	return createHiveColumnStatistics(colStats, hivePartition.getSd(), desc, hiveVersion);
}