Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#setTableName()

/**
 * Convert a {@link HivePartition} into a {@link Partition}.
 */
public static Partition getPartition(HivePartition hivePartition) {
  State props = hivePartition.getProps();
  Partition partition = new Partition();
  partition.setDbName(hivePartition.getDbName());
  partition.setTableName(hivePartition.getTableName());
  partition.setValues(hivePartition.getValues());
  partition.setParameters(getParameters(props));
  if (hivePartition.getCreateTime().isPresent()) {
    partition.setCreateTime(Ints.checkedCast(hivePartition.getCreateTime().get()));
  } else if (props.contains(HiveConstants.CREATE_TIME)) {
    partition.setCreateTime(props.getPropAsInt(HiveConstants.CREATE_TIME));
  }
  if (props.contains(HiveConstants.LAST_ACCESS_TIME)) {
    partition.setLastAccessTime(props.getPropAsInt(HiveConstants.LAST_ACCESS_TIME));
  }
  partition.setSd(getStorageDescriptor(hivePartition));
  return partition;
}
 

/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 

@Test
public void newReplicaPartitionStatisticsWithTransformation() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ReplicaTableFactory factory = new ReplicaTableFactory(SOURCE_META_STORE_URIS, TableTransformation.IDENTITY,
      PartitionTransformation.IDENTITY, COLUMN_STATISTICS_TRANSFORMATION);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is("new_db"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is("new_table"));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("part=newPart"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

static Partition newPartition(Table hiveTable, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

@Before
public void init() {
  partition = new Partition();
  partition.setDbName("database");
  partition.setTableName("table");
  partition.setValues(ImmutableList.of("part"));

  Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>();
  userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo()));
  PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet();
  privileges.setUserPrivileges(userPrivileges);
  partition.setPrivileges(privileges);

  StorageDescriptor storageDescriptor = new StorageDescriptor();
  storageDescriptor.setCols(Arrays.asList(new FieldSchema("a", "int", null)));
  storageDescriptor.setInputFormat("input_format");
  storageDescriptor.setOutputFormat("output_format");
  storageDescriptor.setSerdeInfo(new SerDeInfo("serde", "lib", new HashMap<String, String>()));
  storageDescriptor.setSkewedInfo(new SkewedInfo());
  storageDescriptor.setParameters(new HashMap<String, String>());
  storageDescriptor.setLocation("database/table/part/");
  partition.setSd(storageDescriptor);

  Map<String, String> parameters = new HashMap<>();
  parameters.put("com.company.parameter", "abc");
  partition.setParameters(parameters);
}
 

public static Partition newPartition(String database, String tableName, String partitionValue) {
  Partition partition = new Partition();
  partition.setDbName(database);
  partition.setTableName(tableName);
  partition.setCreateTime(CREATE_TIME);
  partition.setValues(ImmutableList.of(partitionValue));

  Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>();
  userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo()));
  PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet();
  privileges.setUserPrivileges(userPrivileges);
  partition.setPrivileges(privileges);

  StorageDescriptor storageDescriptor = new StorageDescriptor();
  storageDescriptor.setCols(COLS);
  storageDescriptor.setInputFormat(INPUT_FORMAT);
  storageDescriptor.setOutputFormat(OUTPUT_FORMAT);
  storageDescriptor.setSerdeInfo(new SerDeInfo(SERDE_INFO_NAME, SERIALIZATION_LIB, new HashMap<String, String>()));
  storageDescriptor.setSkewedInfo(new SkewedInfo());
  storageDescriptor.setParameters(new HashMap<String, String>());
  storageDescriptor.setLocation(DATABASE + "/" + tableName + "/" + partitionValue + "/");
  partition.setSd(storageDescriptor);

  Map<String, String> parameters = new HashMap<>();
  parameters.put("com.company.parameter", "abc");
  partition.setParameters(parameters);

  return partition;
}
 

private static Partition newPartition(String databaseName, String tableName, String location) {
  Partition partition = new Partition();

  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setParameters(new HashMap<String, String>());
  partition.setValues(Arrays.asList("01"));

  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(location);
  partition.setSd(sd);

  return partition;
}
 

private Partition newPartition(
    String datbase,
    String table,
    StorageDescriptor tableStorageDescriptor,
    List<String> values,
    File location,
    String sourceTable,
    String sourceLocation,
    boolean addChecksum) {
  Partition partition = new Partition();
  partition.setDbName(datbase);
  partition.setTableName(table);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  partition.setParameters(new HashMap<String, String>());
  if (sourceTable != null) {
    partition.getParameters().put(CircusTrainTableParameter.SOURCE_TABLE.parameterName(), sourceTable);
  }
  if (sourceLocation != null) {
    partition.getParameters().put(CircusTrainTableParameter.SOURCE_LOCATION.parameterName(), sourceLocation);
  }
  if (addChecksum) {
    partition.getParameters().put(CircusTrainTableParameter.PARTITION_CHECKSUM.parameterName(), location.getName());
  }
  return partition;
}
 

public static Partition newViewPartition(Table hiveView, List<String> values) {
  Partition partition = new Partition();
  partition.setDbName(hiveView.getDbName());
  partition.setTableName(hiveView.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveView.getSd()));
  partition.getSd().setLocation(null);
  return partition;
}
 

public static Partition newTablePartition(Table hiveTable, List<String> values, URI location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toString());
  return partition;
}
 

public static Partition newPartition(Table table, String... partitionValues) {
  Partition partition = new Partition();
  partition.setTableName(table.getTableName());
  partition.setDbName(table.getDbName());
  partition.setValues(Arrays.asList(partitionValues));
  partition.setSd(table.getSd());
  partition.setParameters(new HashMap<String, String>());
  return partition;
}
 

private Partition newHivePartition(
    String tableName,
    List<String> partitionValues,
    String location,
    Map<String, String> parameters) {
  Partition partition = new Partition();
  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setValues(partitionValues);
  parameters.forEach((key, value) -> partition.putToParameters(key, value));
  partition.putToParameters(DATA_HIGHWAY_VERSION, DataHighwayVersion.VERSION);
  partition.putToParameters(DATA_HIGHWAY_LAST_REVISION, ISO_OFFSET_DATE_TIME.withZone(UTC).format(clock.instant()));
  partition.setSd(AvroStorageDescriptorFactory.create(location));
  return partition;
}
 

@Test
public void newReplicaPartitionStatistics() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

private Partition newPartition(String... values) {
  Partition partition = new Partition();
  partition.setDbName(DB_NAME);
  partition.setTableName(TABLE_NAME);
  StorageDescriptor sd = new StorageDescriptor();
  sd.setLocation(new Path(tableLocation, partitionName(values)).toUri().toString());
  sd.setCols(FIELDS);
  partition.setSd(sd);
  HashMap<String, String> parameters = new HashMap<>();
  parameters.put(StatsSetupConst.ROW_COUNT, "1");
  partition.setParameters(parameters);
  partition.setValues(Arrays.asList(values));
  return partition;
}
 

Partition newReplicaPartition(
    String eventId,
    Table sourceTable,
    Partition sourcePartition,
    String replicaDatabaseName,
    String replicaTableName,
    Path replicaPartitionLocation,
    ReplicationMode replicationMode) {
  Partition replica = partitionTransformation.transform(new Partition(sourcePartition));
  replica.setDbName(replicaDatabaseName);
  replica.setTableName(replicaTableName);
  if (replica.getSd() != null) {
    replica.getSd().setLocation(toStringOrNull(replicaPartitionLocation));
  }

  String sourcePartitionLocation = sourcePartition.getSd() == null ? ""
      : toStringOrEmpty(sourcePartition.getSd().getLocation());

  // Statistic specific parameters
  replica.putToParameters(STATS_GENERATED_VIA_STATS_TASK, Boolean.TRUE.toString());
  replica.putToParameters(STATS_GENERATED, Boolean.TRUE.toString());
  replica.putToParameters(DO_NOT_UPDATE_STATS, Boolean.TRUE.toString());
  // Replication specific parameters
  replica.putToParameters(LAST_REPLICATED.parameterName(), DateTime.now(DateTimeZone.UTC).toString());
  replica.putToParameters(REPLICATION_EVENT.parameterName(), eventId);
  replica.putToParameters(SOURCE_LOCATION.parameterName(), sourcePartitionLocation);
  replica.putToParameters(SOURCE_TABLE.parameterName(), Warehouse.getQualifiedName(sourceTable));
  replica.putToParameters(SOURCE_METASTORE.parameterName(), sourceMetaStoreUris);
  replica.putToParameters(REPLICATION_MODE.parameterName(), replicationMode.name());
  return replica;
}
 

private Partition newPartition(StorageDescriptor tableStorageDescriptor, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(DATABASE);
  partition.setTableName(TABLE);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

/**
 * Creates a Hive partition instance.
 */
public static Partition createHivePartition(String dbName, String tableName, List<String> values,
		StorageDescriptor sd, Map<String, String> parameters) {
	Partition partition = new Partition();
	partition.setDbName(dbName);
	partition.setTableName(tableName);
	partition.setValues(values);
	partition.setParameters(parameters);
	partition.setSd(sd);
	int currentTime = (int) (System.currentTimeMillis() / 1000);
	partition.setCreateTime(currentTime);
	partition.setLastAccessTime(currentTime);
	return partition;
}
 

private Partition makeMetastoreBasePartitionObject(String dbName,
    String tblName, List<String> ptnVals, Table tbl) {
  Partition part4 = new Partition();
  part4.setDbName(dbName);
  part4.setTableName(tblName);
  part4.setValues(ptnVals);
  part4.setParameters(new HashMap<String, String>());
  part4.setSd(tbl.getSd().deepCopy());
  part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
  part4.setParameters(new HashMap<String, String>());
  return part4;
}
 

/**
 * Creates a Hive partition instance.
 */
public static Partition createHivePartition(String dbName, String tableName, List<String> values,
		StorageDescriptor sd, Map<String, String> parameters) {
	Partition partition = new Partition();
	partition.setDbName(dbName);
	partition.setTableName(tableName);
	partition.setValues(values);
	partition.setParameters(parameters);
	partition.setSd(sd);
	int currentTime = (int) (System.currentTimeMillis() / 1000);
	partition.setCreateTime(currentTime);
	partition.setLastAccessTime(currentTime);
	return partition;
}
 

public static Partition convertPartition(com.amazonaws.services.glue.model.Partition src) {
 Partition tgt = new Partition();
 Date createTime = src.getCreationTime();
 if (createTime != null) {
  tgt.setCreateTime((int) (createTime.getTime() / 1000)); 
  tgt.setCreateTimeIsSet(true);
 } else {
  tgt.setCreateTimeIsSet(false);
 }
 String dbName = src.getDatabaseName();
 if (dbName != null) {
  tgt.setDbName(dbName);
  tgt.setDbNameIsSet(true);
 } else {
  tgt.setDbNameIsSet(false);
 }
 Date lastAccessTime = src.getLastAccessTime();
 if (lastAccessTime != null) {
  tgt.setLastAccessTime((int) (lastAccessTime.getTime() / 1000));
  tgt.setLastAccessTimeIsSet(true);
 } else {
  tgt.setLastAccessTimeIsSet(false);
 }
 Map<String, String> params = src.getParameters();
 
 // A null parameter map causes Hive to throw a NPE
 // so ensure we do not return a Partition object with a null parameter map.
 if (params == null) {
   params = Maps.newHashMap();
 }
 
 tgt.setParameters(params);
 tgt.setParametersIsSet(true);
 
 String tableName = src.getTableName();
 if (tableName != null) {
  tgt.setTableName(tableName);
  tgt.setTableNameIsSet(true);
 } else {
  tgt.setTableNameIsSet(false);
 }
 
 List<String> values = src.getValues();
 if (values != null) {
  tgt.setValues(values);
  tgt.setValuesIsSet(true);
 } else {
  tgt.setValuesIsSet(false);
 }
 
 com.amazonaws.services.glue.model.StorageDescriptor sd = src.getStorageDescriptor();
 if (sd != null) {
  StorageDescriptor hiveSd = convertStorageDescriptor(sd);
  tgt.setSd(hiveSd);
  tgt.setSdIsSet(true);
 } else {
  tgt.setSdIsSet(false);
 }
 
 return tgt;
}