Java Code Examples for org.apache.hadoop.hive.metastore.api.Partition#setDbName()

/**
 * Copies partitions from oldTable to newTable, partitions copied are modified to take the schema of newTable
 */
public void execute(CloseableMetaStoreClient client, Table oldTable, Table newTable) throws TException {
  int count = 0;
  String databaseName = newTable.getDbName();
  String tableName = newTable.getTableName();
  PartitionIterator partitionIterator = new PartitionIterator(client, oldTable, partitionBatchSize);
  while (partitionIterator.hasNext()) {
    List<Partition> batch = new ArrayList<>();
    for (int i = 0; i < partitionBatchSize && partitionIterator.hasNext(); i++) {
      Partition partition = partitionIterator.next();
      count++;
      Partition copy = new Partition(partition);
      copy.setDbName(databaseName);
      copy.setTableName(tableName);
      StorageDescriptor sd = new StorageDescriptor(partition.getSd());
      sd.setCols(newTable.getSd().getCols());
      copy.setSd(sd);
      batch.add(copy);
    }
    LOG.info("Copying batch of size {} to {}.{}", batch.size(), databaseName, tableName);
    client.add_partitions(batch);
  }
  LOG.info("Copied {} partitions to {}.{}", count, databaseName, tableName);
}
 

@Test
public void add_partitions_req() throws TException {
  Partition newPartition1 = new Partition();
  newPartition1.setDbName(DB_P);
  Partition newPartition2 = new Partition();
  newPartition2.setDbName(DB_P);
  List<Partition> partitions = Lists.newArrayList(newPartition1, newPartition2);
  AddPartitionsRequest request = new AddPartitionsRequest();
  request.setDbName(DB_P);
  request.setParts(partitions);
  AddPartitionsRequest inbound = new AddPartitionsRequest();
  AddPartitionsResult addPartitionResult = new AddPartitionsResult();
  AddPartitionsResult outbound = new AddPartitionsResult();
  when(primaryMapping.transformInboundAddPartitionsRequest(request)).thenReturn(inbound);
  when(primaryClient.add_partitions_req(inbound)).thenReturn(addPartitionResult);
  when(primaryMapping.transformOutboundAddPartitionsResult(addPartitionResult)).thenReturn(outbound);

  AddPartitionsResult result = handler.add_partitions_req(request);
  assertThat(result, is(outbound));
  verify(primaryMapping, times(3)).checkWritePermissions(DB_P);
}
 

@Test
public void newReplicaPartitionStatistics() throws MetaException {
  sourceTable.setPartitionKeys(
      Arrays.asList(new FieldSchema("one", "string", null), new FieldSchema("two", "string", null)));

  Partition replicaPartition = new Partition(sourcePartition);
  replicaPartition.setDbName(MAPPED_DB_NAME);
  replicaPartition.setTableName(MAPPED_TABLE_NAME);
  replicaPartition.setValues(Arrays.asList("A", "B"));

  ColumnStatisticsObj columnStatisticsObj1 = new ColumnStatisticsObj();
  ColumnStatisticsObj columnStatisticsObj2 = new ColumnStatisticsObj();
  List<ColumnStatisticsObj> columnStatisticsObjs = Arrays.asList(columnStatisticsObj1, columnStatisticsObj2);

  ColumnStatisticsDesc columnStatisticsDesc = new ColumnStatisticsDesc(false, DB_NAME, TABLE_NAME);
  columnStatisticsDesc
      .setPartName(Warehouse.makePartName(sourceTable.getPartitionKeys(), replicaPartition.getValues()));

  ColumnStatistics sourcePartitionStatistics = new ColumnStatistics(columnStatisticsDesc, columnStatisticsObjs);

  ColumnStatistics replicaPartitionStatistics = factory.newReplicaPartitionStatistics(sourceTable, replicaPartition,
      sourcePartitionStatistics);

  assertThat(replicaPartitionStatistics.getStatsDesc().getDbName(), is(MAPPED_DB_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getTableName(), is(MAPPED_TABLE_NAME));
  assertThat(replicaPartitionStatistics.getStatsDesc().getPartName(), is("one=A/two=B"));
  assertThat(replicaPartitionStatistics.getStatsObj().size(), is(2));
  assertThat(replicaPartitionStatistics.getStatsObj().get(0), is(columnStatisticsObj1));
  assertThat(replicaPartitionStatistics.getStatsObj().get(1), is(columnStatisticsObj2));
}
 

@Before
public void init() {
  partition = new Partition();
  partition.setDbName("database");
  partition.setTableName("table");
  partition.setValues(ImmutableList.of("part"));

  Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>();
  userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo()));
  PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet();
  privileges.setUserPrivileges(userPrivileges);
  partition.setPrivileges(privileges);

  StorageDescriptor storageDescriptor = new StorageDescriptor();
  storageDescriptor.setCols(Arrays.asList(new FieldSchema("a", "int", null)));
  storageDescriptor.setInputFormat("input_format");
  storageDescriptor.setOutputFormat("output_format");
  storageDescriptor.setSerdeInfo(new SerDeInfo("serde", "lib", new HashMap<String, String>()));
  storageDescriptor.setSkewedInfo(new SkewedInfo());
  storageDescriptor.setParameters(new HashMap<String, String>());
  storageDescriptor.setLocation("database/table/part/");
  partition.setSd(storageDescriptor);

  Map<String, String> parameters = new HashMap<>();
  parameters.put("com.company.parameter", "abc");
  partition.setParameters(parameters);
}
 

public static Partition newPartition(String database, String tableName, String partitionValue) {
  Partition partition = new Partition();
  partition.setDbName(database);
  partition.setTableName(tableName);
  partition.setCreateTime(CREATE_TIME);
  partition.setValues(ImmutableList.of(partitionValue));

  Map<String, List<PrivilegeGrantInfo>> userPrivileges = new HashMap<>();
  userPrivileges.put("read", ImmutableList.of(new PrivilegeGrantInfo()));
  PrincipalPrivilegeSet privileges = new PrincipalPrivilegeSet();
  privileges.setUserPrivileges(userPrivileges);
  partition.setPrivileges(privileges);

  StorageDescriptor storageDescriptor = new StorageDescriptor();
  storageDescriptor.setCols(COLS);
  storageDescriptor.setInputFormat(INPUT_FORMAT);
  storageDescriptor.setOutputFormat(OUTPUT_FORMAT);
  storageDescriptor.setSerdeInfo(new SerDeInfo(SERDE_INFO_NAME, SERIALIZATION_LIB, new HashMap<String, String>()));
  storageDescriptor.setSkewedInfo(new SkewedInfo());
  storageDescriptor.setParameters(new HashMap<String, String>());
  storageDescriptor.setLocation(DATABASE + "/" + tableName + "/" + partitionValue + "/");
  partition.setSd(storageDescriptor);

  Map<String, String> parameters = new HashMap<>();
  parameters.put("com.company.parameter", "abc");
  partition.setParameters(parameters);

  return partition;
}
 

static Partition newPartition(Table hiveTable, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

private void add_partition(HiveMetaStoreClient client, Table table,
	      List<String> vals, String location) throws InvalidObjectException,
	        AlreadyExistsException, MetaException, TException {

    Partition part = new Partition();
    part.setDbName(table.getDbName());
    part.setTableName(table.getTableName());
    part.setValues(vals);
    part.setParameters(new HashMap<String, String>());
    part.setSd(table.getSd());
    part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
    part.getSd().setLocation(table.getSd().getLocation() + location);

    client.add_partition(part);
}
 

public static Partition newViewPartition(Table hiveView, List<String> values) {
  Partition partition = new Partition();
  partition.setDbName(hiveView.getDbName());
  partition.setTableName(hiveView.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveView.getSd()));
  partition.getSd().setLocation(null);
  return partition;
}
 

public static Partition newTablePartition(Table hiveTable, List<String> values, URI location) {
  Partition partition = new Partition();
  partition.setDbName(hiveTable.getDbName());
  partition.setTableName(hiveTable.getTableName());
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(hiveTable.getSd()));
  partition.getSd().setLocation(location.toString());
  return partition;
}
 

private Partition newHivePartition(
    String tableName,
    List<String> partitionValues,
    String location,
    Map<String, String> parameters) {
  Partition partition = new Partition();
  partition.setDbName(databaseName);
  partition.setTableName(tableName);
  partition.setValues(partitionValues);
  parameters.forEach((key, value) -> partition.putToParameters(key, value));
  partition.putToParameters(DATA_HIGHWAY_VERSION, DataHighwayVersion.VERSION);
  partition.putToParameters(DATA_HIGHWAY_LAST_REVISION, ISO_OFFSET_DATE_TIME.withZone(UTC).format(clock.instant()));
  partition.setSd(AvroStorageDescriptorFactory.create(location));
  return partition;
}
 

@Before
public void setUp() {
  databaseMapping = new DatabaseMappingImpl(metastoreMapping, queryMapping);
  database = new Database();
  database.setName(DB_NAME);
  partition = new Partition();
  partition.setDbName(DB_NAME);
  partitions = Lists.newArrayList(partition);
  index = new Index();
  index.setDbName(DB_NAME);
  hiveObjectRef = new HiveObjectRef();
  hiveObjectRef.setDbName(DB_NAME);
  hiveObjectRef.setObjectType(HiveObjectType.DATABASE);
  hiveObjectRef.setObjectName(DB_NAME);
  hiveObjectPrivileges = new ArrayList<>();
  HiveObjectPrivilege hiveObjectPrivilege = new HiveObjectPrivilege();
  hiveObjectPrivilege.setHiveObject(hiveObjectRef);
  hiveObjectPrivileges.add(hiveObjectPrivilege);
  partitionSpec = new PartitionSpec();
  partitionSpec.setDbName(DB_NAME);
  when(metastoreMapping.transformInboundDatabaseName(anyString())).thenReturn(IN_DB_NAME);
  when(metastoreMapping.transformOutboundDatabaseName(anyString())).thenReturn(OUT_DB_NAME);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_EXPANDED_TEXT))
      .thenReturn(VIEW_EXPANDED_TEXT_TRANSFORMED);
  when(queryMapping.transformOutboundDatabaseName(metastoreMapping, VIEW_ORIGINAL_TEXT))
      .thenReturn(VIEW_ORIGINAL_TEXT_TRANSFORMED);
}
 

Partition newReplicaPartition(
    String eventId,
    Table sourceTable,
    Partition sourcePartition,
    String replicaDatabaseName,
    String replicaTableName,
    Path replicaPartitionLocation,
    ReplicationMode replicationMode) {
  Partition replica = partitionTransformation.transform(new Partition(sourcePartition));
  replica.setDbName(replicaDatabaseName);
  replica.setTableName(replicaTableName);
  if (replica.getSd() != null) {
    replica.getSd().setLocation(toStringOrNull(replicaPartitionLocation));
  }

  String sourcePartitionLocation = sourcePartition.getSd() == null ? ""
      : toStringOrEmpty(sourcePartition.getSd().getLocation());

  // Statistic specific parameters
  replica.putToParameters(STATS_GENERATED_VIA_STATS_TASK, Boolean.TRUE.toString());
  replica.putToParameters(STATS_GENERATED, Boolean.TRUE.toString());
  replica.putToParameters(DO_NOT_UPDATE_STATS, Boolean.TRUE.toString());
  // Replication specific parameters
  replica.putToParameters(LAST_REPLICATED.parameterName(), DateTime.now(DateTimeZone.UTC).toString());
  replica.putToParameters(REPLICATION_EVENT.parameterName(), eventId);
  replica.putToParameters(SOURCE_LOCATION.parameterName(), sourcePartitionLocation);
  replica.putToParameters(SOURCE_TABLE.parameterName(), Warehouse.getQualifiedName(sourceTable));
  replica.putToParameters(SOURCE_METASTORE.parameterName(), sourceMetaStoreUris);
  replica.putToParameters(REPLICATION_MODE.parameterName(), replicationMode.name());
  return replica;
}
 

private Partition newPartition(StorageDescriptor tableStorageDescriptor, List<String> values, File location) {
  Partition partition = new Partition();
  partition.setDbName(DATABASE);
  partition.setTableName(TABLE);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

private Partition newPartition(
    String table,
    StorageDescriptor tableStorageDescriptor,
    List<String> values,
    File location) {
  Partition partition = new Partition();
  partition.setDbName(DATABASE);
  partition.setTableName(table);
  partition.setValues(values);
  partition.setSd(new StorageDescriptor(tableStorageDescriptor));
  partition.getSd().setLocation(location.toURI().toString());
  return partition;
}
 

@Test
public void dropPartition() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  HiveMetaStoreClient client = server.getCore().newClient();

  try {
    Table table = createPartitionedTable(DATABASE, tableName, server);

    Partition partition = new Partition();
    partition.setDbName(DATABASE);
    partition.setTableName(tableName);
    partition.setValues(Arrays.asList("1"));
    partition.setSd(new StorageDescriptor(table.getSd()));
    partition.getSd().setLocation(
        String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName));
    client.add_partition(partition);

    try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
        Statement statement = connection.createStatement()) {
      String dropPartitionHql = String.format("ALTER TABLE %s.%s DROP PARTITION (partcol=1)", DATABASE, tableName);
      statement.execute(dropPartitionHql);
    }

    List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1);
    assertThat(partitions.size(), is(0));
  } finally {
    client.close();
  }
  server.shutdown();
}
 

@Test
public void alter_partitions() throws TException {
  Partition newPartition1 = new Partition();
  newPartition1.setDbName(DB_P);
  Partition newPartition2 = new Partition();
  newPartition2.setDbName(DB_P);
  List<Partition> inbound = Lists.newArrayList(new Partition());
  List<Partition> partitions = Lists.newArrayList(newPartition1, newPartition2);
  when(primaryMapping.transformInboundPartitions(partitions)).thenReturn(inbound);
  handler.alter_partitions(DB_P, "table", partitions);
  verify(primaryMapping, times(3)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_partitions(DB_P, "table", inbound);
}
 

private Partition makeMetastoreBasePartitionObject(String dbName,
    String tblName, List<String> ptnVals, Table tbl) {
  Partition part4 = new Partition();
  part4.setDbName(dbName);
  part4.setTableName(tblName);
  part4.setValues(ptnVals);
  part4.setParameters(new HashMap<String, String>());
  part4.setSd(tbl.getSd().deepCopy());
  part4.getSd().setSerdeInfo(tbl.getSd().getSerdeInfo().deepCopy());
  part4.setParameters(new HashMap<String, String>());
  return part4;
}
 

@Override
public Partition transformOutboundPartition(Partition partition) {
  partition.setDbName(metaStoreMapping.transformOutboundDatabaseName(partition.getDbName()));
  return partition;
}
 

/**
 * {@inheritDoc}
 */
@Override
public Partition metacatToHivePartition(final PartitionDto partitionDto, @Nullable final TableDto tableDto) {
    final Partition result = new Partition();

    final QualifiedName name = partitionDto.getName();
    List<String> values = Lists.newArrayListWithCapacity(16);
    String databaseName = null;
    String tableName = null;
    if (name != null) {
        if (name.getPartitionName() != null) {
            //
            // Unescape the partition name to get the right partition values.
            // Partition name always are escaped where as the parition values are not.
            //
            values = getPartValsFromName(tableDto, name.getPartitionName());
        }

        if (name.getDatabaseName() != null) {
            databaseName = name.getDatabaseName();
        }

        if (name.getTableName() != null) {
            tableName = name.getTableName();
        }
    }
    result.setValues(values);
    result.setDbName(databaseName);
    result.setTableName(tableName);

    Map<String, String> metadata = partitionDto.getMetadata();
    if (metadata == null) {
        metadata = Maps.newHashMap();
    }
    result.setParameters(metadata);

    result.setSd(fromStorageDto(partitionDto.getSerde(), tableName));
    final StorageDescriptor sd = result.getSd();
    if (tableDto != null) {
        if (sd.getSerdeInfo() != null && tableDto.getSerde() != null && Strings.isNullOrEmpty(
            sd.getSerdeInfo().getSerializationLib())) {
            sd.getSerdeInfo().setSerializationLib(tableDto.getSerde().getSerializationLib());
        }

        final List<FieldDto> fields = tableDto.getFields();
        if (fields == null) {
            sd.setCols(Collections.emptyList());
        } else {
            sd.setCols(fields.stream()
                .filter(field -> !field.isPartition_key())
                .map(this::metacatToHiveField)
                .collect(Collectors.toList()));
        }
    }

    final AuditDto auditDto = partitionDto.getAudit();
    if (auditDto != null) {
        if (auditDto.getCreatedDate() != null) {
            result.setCreateTime(dateToEpochSeconds(auditDto.getCreatedDate()));
        }
        if (auditDto.getLastModifiedDate() != null) {
            result.setLastAccessTime(dateToEpochSeconds(auditDto.getLastModifiedDate()));
        }
    }

    return result;
}
 

@Override
public Partition transformInboundPartition(Partition partition) {
  partition.setDbName(metaStoreMapping.transformInboundDatabaseName(partition.getDbName()));
  return partition;
}