Java Code Examples for org.apache.hadoop.hive.metastore.Warehouse#makePartName()

/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
 

@Override
public org.apache.hadoop.hive.metastore.api.Partition getPartitionWithAuthInfo(
      String databaseName, String tableName, List<String> values,
      String userName, List<String> groupNames)
      throws MetaException, UnknownTableException, NoSuchObjectException, TException {

    // TODO move this into the service
    org.apache.hadoop.hive.metastore.api.Partition partition = getPartition(databaseName, tableName, values);
    org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
    if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
        String partName = Warehouse.makePartName(table.getPartitionKeys(), values);
        HiveObjectRef obj = new HiveObjectRef();
        obj.setObjectType(HiveObjectType.PARTITION);
        obj.setDbName(databaseName);
        obj.setObjectName(tableName);
        obj.setPartValues(values);
        org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet =
              this.get_privilege_set(obj, userName, groupNames);
        partition.setPrivileges(privilegeSet);
    }

    return partition;
}
 

@Test
public void testAppendPartitionByName() throws Exception {    
  List<String> values = Arrays.asList("foo");
  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  Path partLocation = new Path(testTable.getSd().getLocation(), Warehouse
          .makePartName(testTable.getPartitionKeys(), values));
  setupMockWarehouseForPath(partLocation, false, true);
  mockBatchCreatePartitionsSucceed();

  org.apache.hadoop.hive.metastore.api.Partition res = metastoreClient.appendPartition(
      testDB.getName(),
      testTable.getTableName(),
      testTable.getPartitionKeys().get(0).getName() + "=foo");
  assertThat(res.getValues(), is(values));
  assertDaemonThreadPools();
}
 

@Override
public org.apache.hadoop.hive.metastore.api.Partition getPartitionWithAuthInfo(
      String databaseName, String tableName, List<String> values,
      String userName, List<String> groupNames)
      throws MetaException, UnknownTableException, NoSuchObjectException, TException {

  // TODO move this into the service
  org.apache.hadoop.hive.metastore.api.Partition partition = getPartition(databaseName, tableName, values);
  org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
  if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
    String partName = Warehouse.makePartName(table.getPartitionKeys(), values);
    HiveObjectRef obj = new HiveObjectRef();
    obj.setObjectType(HiveObjectType.PARTITION);
    obj.setDbName(databaseName);
    obj.setObjectName(tableName);
    obj.setPartValues(values);
    org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet =
          this.get_privilege_set(obj, userName, groupNames);
    partition.setPrivileges(privilegeSet);
  }

  return partition;
}
 

@Test
public void testAppendPartition() throws Exception {
  List<String> values = ImmutableList.of("foo");
  when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
  Path partLocation = new Path(testTbl.getStorageDescriptor().getLocation(),
    Warehouse.makePartName(CatalogToHiveConverter.convertFieldSchemaList(testTbl.getPartitionKeys()), values));
  setupMockWarehouseForPath(partLocation, false, true);
  mockBatchCreatePartitionsSucceed();

  org.apache.hadoop.hive.metastore.api.Partition res =
    metastoreClientDelegate.appendPartition(testDb.getName(), testTbl.getName(), values);

  verify(wh, times(1)).mkdirs(partLocation, true);
  assertThat(res.getValues(), is(values));
}
 

private Path getDestinationPathForRename(String dbName, org.apache.hadoop.hive.metastore.api.Table tbl, org.apache.hadoop.hive.metastore.api.Partition newPartition)
      throws InvalidOperationException, MetaException, TException {
    try {
        Path destPath = new Path(hiveShims.getDefaultTablePath(getDatabase(dbName), tbl.getTableName(), wh),
              Warehouse.makePartName(tbl.getPartitionKeys(), newPartition.getValues()));
        return constructRenamedPath(destPath, new Path(newPartition.getSd().getLocation()));
    } catch (NoSuchObjectException e) {
        throw new InvalidOperationException(
              "Unable to change partition or table. Database " + dbName + " does not exist"
                    + " Check metastore logs for detailed stack." + e.getMessage());
    }
}
 

@Test
public void testRenamePartitionForHiveManagedTable() throws Exception {
  String dbName = testDB.getName();
  String tblName = testTable.getTableName();
  List<String> partitionValues = testPartition.getValues();
  Partition catalogPartition = HiveToCatalogConverter.convertPartition(testPartition);
  
  Partition newPartition = new Partition()
                                .withDatabaseName(dbName).withTableName(tblName)
                                .withValues(Lists.newArrayList("newval"))
                                .withStorageDescriptor(catalogPartition.getStorageDescriptor());
  
  Path srcPath = new Path(testPartition.getSd().getLocation());
  Path expectedDestPath = new Path("/db/tbl/" ,
      Warehouse.makePartName(testTable.getPartitionKeys(), newPartition.getValues()));
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(catalogPartition));

  FileSystem fs = mock(FileSystem.class);
  when(fs.getUri()).thenReturn(new URI("s3://bucket"));
  when(hiveShims.getDefaultTablePath(any(org.apache.hadoop.hive.metastore.api.Database.class), anyString(), wh))
      .thenReturn(new Path("/db/tbl"));
  when(wh.getFs(any(Path.class))).thenReturn(fs);
  when(fs.exists(srcPath)).thenReturn(true);
  when(fs.exists(expectedDestPath)).thenReturn(false);
  when(wh.mkdirs(expectedDestPath.getParent(), true)).thenReturn(true);       
  
  metastoreClient.renamePartition(dbName, tblName, partitionValues,
      CatalogToHiveConverter.convertPartition(newPartition));
  
  // Verify catalog service is called , dest path's parent dirs are created and dfs rename is done
  verify(glueClient, times(1)).updatePartition(any(UpdatePartitionRequest.class));
  verify(wh).mkdirs(expectedDestPath.getParent(), true);
  verify(wh).renameDir(srcPath, expectedDestPath, true);
}
 

@Test(expected=InvalidOperationException.class)
public void testRenamePartitionDestinationAlreadyExists() throws Exception {
  String dbName = testDB.getName();
  String tblName = testTable.getTableName();
  List<String> partitionValues = testPartition.getValues();
  Partition catalogPartition = HiveToCatalogConverter.convertPartition(testPartition);
  
  Partition newPartition = new Partition()
                                .withDatabaseName(dbName).withTableName(tblName)
                                .withValues(Lists.newArrayList("newval"))
                                .withStorageDescriptor(catalogPartition.getStorageDescriptor());
  
  Path srcPath = new Path(testPartition.getSd().getLocation());
  Path expectedDestPath = new Path("/db/tbl/" , Warehouse.makePartName(testTable.getPartitionKeys(),
      newPartition.getValues()));
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(catalogPartition));
  
  
  FileSystem fs = mock(FileSystem.class);
  when(fs.getUri()).thenReturn(new URI("s3://bucket"));
  when(hiveShims.getDefaultTablePath(any(org.apache.hadoop.hive.metastore.api.Database.class), anyString(), wh))
      .thenReturn(new Path("/db/tbl"));
  when(wh.getFs(any(Path.class))).thenReturn(fs);
  when(fs.exists(srcPath)).thenReturn(true);
  when(fs.exists(expectedDestPath)).thenReturn(true);
      
  metastoreClient.renamePartition(dbName, tblName, partitionValues,
      CatalogToHiveConverter.convertPartition(newPartition));
}
 

private Path getDestinationPathForRename(String dbName, org.apache.hadoop.hive.metastore.api.Table tbl, org.apache.hadoop.hive.metastore.api.Partition newPartition)
      throws InvalidOperationException, MetaException, TException {
  try {
    Path destPath = new Path(hiveShims.getDefaultTablePath(getDatabase(dbName), tbl.getTableName(), wh),
          Warehouse.makePartName(tbl.getPartitionKeys(), newPartition.getValues()));
    return constructRenamedPath(destPath, new Path(newPartition.getSd().getLocation()));
  } catch (NoSuchObjectException e) {
    throw new InvalidOperationException(
          "Unable to change partition or table. Database " + dbName + " does not exist"
                + " Check metastore logs for detailed stack." + e.getMessage());
  }
}
 

private static String getPartitionName(List<FieldSchema> partitionKeys, Partition partition) {
  try {
    return Warehouse.makePartName(partitionKeys, partition.getValues());
  } catch (MetaException e) {
    throw new RuntimeException(e);
  }
}
 

private void dropAndDeletePartitions(CloseableMetaStoreClient client, Predicate<String> shouldDelete)
  throws MetaException, TException, NoSuchObjectException {
  Table replicaTable = client.getTable(databaseName, tableName);
  List<FieldSchema> partitionKeys = replicaTable.getPartitionKeys();
  if (partitionKeys == null || partitionKeys.isEmpty()) {
    // unpartitioned table nothing to delete
    return;
  }
  PartitionIterator partitionIterator = new PartitionIterator(client, replicaTable, (short) 1000);
  while (partitionIterator.hasNext()) {
    Partition replicaPartition = partitionIterator.next();
    List<String> values = replicaPartition.getValues();
    String partitionName = Warehouse.makePartName(partitionKeys, values);
    if (shouldDelete.apply(partitionName)) {
      log
          .info("Dropping partition for replica table: "
              + databaseName
              + "."
              + tableName
              + ", partition value: '"
              + partitionName
              + "'");
      client.dropPartition(databaseName, tableName, partitionName, DELETE_DATA);
      Path oldLocation = locationAsPath(replicaPartition);
      String oldEventId = replicaPartition.getParameters().get(REPLICATION_EVENT.parameterName());
      cleanupLocationManager.addCleanupLocation(oldEventId, oldLocation);
    }
  }
}
 

private String partitionName(String... values) {
  try {
    return Warehouse.makePartName(PARTITIONS, Arrays.asList(values));
  } catch (MetaException e) {
    throw new RuntimeException(e);
  }
}
 

private static String partitionName(Table table, Partition partition) {
  try {
    return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
  } catch (MetaException e) {
    throw new CircusTrainException("Unable to build partition name for partition values "
        + partition.getValues()
        + " of table "
        + Warehouse.getQualifiedName(table), e);
  }
}
 

/**
 * Generate partition name from the <code>partValues</code>.
 *
 * @param partitionKeys list of partition keys
 * @param partValues list of partition values
 * @return partition name
 */
public static String makePartName(final List<FieldSchema> partitionKeys, final List<String> partValues) {
    try {
        return Warehouse.makePartName(partitionKeys, partValues);
    } catch (MetaException e) {
        throw new InvalidMetaException("Failed making the part name from the partition values", e);
    }
}
 

private String getNameOfPartition(final Table table, final Partition partition) {
    try {
        return Warehouse.makePartName(table.getPartitionKeys(), partition.getValues());
    } catch (TException e) {
        throw new InvalidMetaException("One or more partition names are invalid.", e);
    }
}
 

@Test
public void testRenamePartitionRevertWhenMetastoreoperationFails() throws Exception {
  String dbName = testDB.getName();
  String tblName = testTable.getTableName();
  List<String> partitionValues = testPartition.getValues();
  Partition catalogPartition = HiveToCatalogConverter.convertPartition(testPartition);
  
  Partition newPartition = new Partition()
                              .withDatabaseName(dbName).withTableName(tblName)
                              .withValues(Lists.newArrayList("newval"))
                              .withStorageDescriptor(catalogPartition.getStorageDescriptor());
  
  Path srcPath = new Path(testPartition.getSd().getLocation());
  Path expectedDestPath = new Path("/db/tbl/" ,
      Warehouse.makePartName(testTable.getPartitionKeys(), newPartition.getValues()));
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(catalogPartition));

  FileSystem fs = mock(FileSystem.class);
  when(fs.getUri()).thenReturn(new URI("s3://bucket"));
  when(hiveShims.getDefaultTablePath(any(org.apache.hadoop.hive.metastore.api.Database.class), anyString(), wh))
      .thenReturn(new Path("/db/tbl"));
  when(wh.getFs(any(Path.class))).thenReturn(fs);
  when(fs.exists(srcPath)).thenReturn(true);
  when(fs.exists(expectedDestPath)).thenReturn(false);
  
  // Fail directory creation
  when(wh.mkdirs(expectedDestPath.getParent(), true)).thenReturn(false);
  
  boolean exceptionThrown = false;
  try {
    metastoreClient.renamePartition(dbName, tblName, partitionValues,
        CatalogToHiveConverter.convertPartition(newPartition));
  } catch(InvalidOperationException e) {
    exceptionThrown = true;
  }
  assertTrue(exceptionThrown);
  
  // Verify catalog service is called , dest path's parent dirs are created and dfs rename is done
  verify(glueClient, times(2)).updatePartition(any(UpdatePartitionRequest.class));
}