org.apache.hadoop.hive.metastore.api.InvalidObjectException Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.InvalidObjectException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchDeletePartitionsHelperTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeletePartitionsThrowsInvalidInputException() throws Exception {
  Exception e = new com.amazonaws.services.glue.model.InvalidInputException("foo");
  mockBatchDeleteThrowsException(e);

  List<String> values1 = Lists.newArrayList("val1");
  List<String> values2 = Lists.newArrayList("val2");
  List<Partition> partitions = Lists.newArrayList(
        TestObjects.getTestPartition(NAMESPACE_NAME, TABLE_NAME, values1),
        TestObjects.getTestPartition(NAMESPACE_NAME, TABLE_NAME, values2));
  batchDeletePartitionsHelper = new BatchDeletePartitionsHelper(client, NAMESPACE_NAME, TABLE_NAME, null, partitions);

  batchDeletePartitionsHelper.deletePartitions();
  assertTrue(batchDeletePartitionsHelper.getPartitionsDeleted().isEmpty());
  assertThat(batchDeletePartitionsHelper.getFirstTException(), is(instanceOf(InvalidObjectException.class)));
}
 
Example #2
Source File: AWSCatalogMetastoreClientTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test
public void testDropPartitionsClientException() throws Exception {
  Table table = HiveToCatalogConverter.convertTable(testTable);
  String namespaceName = testDB.getName();
  String tableName = table.getName();
  Partition partition = TestObjects.getTestPartition(namespaceName, tableName, Arrays.asList("foo", "bar"));

  mockGetPartitionsSuccess(Lists.newArrayList(partition));
  mockBatchDeleteThrowsException(new InvalidInputException("InvalidInputException"));
  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));

  try {
    metastoreClient.dropPartitions(namespaceName, tableName,
        Lists.newArrayList(getDumbExpression()), true, false, false);
    fail("should throw");
  } catch (TException e) {
    verify(glueClient, times(1)).batchDeletePartition(any(BatchDeletePartitionRequest.class));
    verify(glueClient, never()).getPartition(any(GetPartitionRequest.class));
    verify(wh, never()).deleteDir(any(Path.class), anyBoolean(), anyBoolean());
    assertThat(e, is(instanceOf(InvalidObjectException.class)));
    assertThat(e.getMessage(), containsString("InvalidInputException"));
  }
  assertDaemonThreadPools();
}
 
Example #3
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String source_db,
    String source_table_name,
    String dest_db,
    String dest_table_name)
    throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(source_db);
  mapping.checkWritePermissions(dest_db);
  Partition result = mapping
      .getClient()
      .exchange_partition(partitionSpecs, mapping.transformInboundDatabaseName(source_db), source_table_name,
          mapping.transformInboundDatabaseName(dest_db), dest_table_name);
  return mapping.transformOutboundPartition(result);
}
 
Example #4
Source File: MetastoreClientUtils.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
/**
 * Taken from HiveMetaStore#create_table_core
 * https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java#L1370-L1383
 */
public static void validateTableObject(Table table, Configuration conf) throws InvalidObjectException {
  checkNotNull(table, "table cannot be null");
  checkNotNull(table.getSd(), "Table#StorageDescriptor cannot be null");

  if (!hiveShims.validateTableName(table.getTableName(), conf)) {
    throw new InvalidObjectException(table.getTableName() + " is not a valid object name");
  }
  String validate = MetaStoreUtils.validateTblColumns(table.getSd().getCols());
  if (validate != null) {
    throw new InvalidObjectException("Invalid column " + validate);
  }

  if (table.getPartitionKeys() != null) {
    validate = MetaStoreUtils.validateTblColumns(table.getPartitionKeys());
    if (validate != null) {
      throw new InvalidObjectException("Invalid partition column " + validate);
    }
  }
}
 
Example #5
Source File: GlueMetastoreClientDelegateTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test
public void testAddPartitionsThrowsExceptionSecondPage() throws Exception {
  int numPartitions = 200;
  int secondPageSize = numPartitions - BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE;
  when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
    .thenReturn(new BatchCreatePartitionResult())
    .thenThrow(new InvalidInputException("exception"));
  when(glueClient.getTable(any(GetTableRequest.class)))
    .thenReturn(new GetTableResult().withTable(testTbl));

  List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);

  try {
    metastoreClientDelegate.addPartitions(partitions, false, true);
    fail("Should throw");
  } catch (Exception e) {
    assertThat(e, is(instanceOf(InvalidObjectException.class)));
    verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
    verify(glueClient, times(2)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
    verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
    verify(wh, times(secondPageSize)).deleteDir(any(Path.class), eq(true));
    assertDaemonThreadPools();
  }
}
 
Example #6
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<Partition> exchange_partitions(
    Map<String, String> partitionSpecs,
    String source_db,
    String source_table_name,
    String dest_db,
    String dest_table_name)
    throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(source_db);
  mapping.checkWritePermissions(dest_db);
  List<Partition> result = mapping
      .getClient()
      .exchange_partitions(partitionSpecs, mapping.transformInboundDatabaseName(source_db), source_table_name,
          mapping.transformInboundDatabaseName(dest_db), dest_table_name);
  return mapping.transformOutboundPartitions(result);
}
 
Example #7
Source File: CircusTrainReplicationModeIntegrationTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private void addPartitionsToSource(Table sourceTable, URI sourceTableLocation)
  throws IOException, InvalidObjectException, AlreadyExistsException, MetaException, TException {

  final File dataFileUk = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileUk, "1\tadam\tlondon\n2\tsusan\tglasgow\n");
  String partitionEurope = "/continent=Europe";
  String partitionUk = partitionEurope + "/country=UK";
  String partitionFileLocationUk = SOURCE_MANAGED_PARTITIONED_TABLE + partitionUk;
  String fileKeyUk = String.format("%s/%s/%s", DATABASE, partitionFileLocationUk, PART_00000);
  s3Client.putObject("source", fileKeyUk, dataFileUk);

  final File dataFileChina = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileChina, "1\tchun\tbeijing\n2\tshanghai\tmilan\n");
  String partitionAsia = "/continent=Asia";
  String partitionChina = partitionAsia + "/country=China";
  String partitionFileLocation = SOURCE_MANAGED_PARTITIONED_TABLE + partitionChina;
  String fileKeyChina = String.format("%s/%s/%s", DATABASE, partitionFileLocation, PART_00000);
  s3Client.putObject("source", fileKeyChina, dataFileChina);

  URI ukPartitionURI = URI.create(sourceTableLocation + partitionUk);
  URI chinaPartitionURI = URI.create(sourceTableLocation + partitionChina);
  sourceCatalog
      .client()
      .add_partitions(Arrays
          .asList(newTablePartition(sourceTable, Arrays.asList("Europe", "UK"), ukPartitionURI),
              newTablePartition(sourceTable, Arrays.asList("Asia", "China"), chinaPartitionURI)));
}
 
Example #8
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_environment_context(Table tbl, EnvironmentContext environment_context)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_environment_context(mapping.transformInboundTable(tbl), environment_context);
}
 
Example #9
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table(Table tbl)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table(mapping.transformInboundTable(tbl));
}
 
Example #10
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_database(Database database)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.primaryDatabaseMapping();
  mapping.createDatabase(mapping.transformInboundDatabase(database));
}
 
Example #11
Source File: MetaStoreMappingImpl.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public void createDatabase(Database database)
  throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  if (accessControlHandler.hasCreatePermission()) {
    getClient().create_database(database);
    accessControlHandler.databaseCreatedNotification(database.getName());
  } else {
    throw new NotAllowedException("You cannot create the database '" + database.getName() + "'.");
  }
}
 
Example #12
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}
 
Example #13
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition_with_environment_context(Partition new_part, EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping
      .getClient()
      .add_partition_with_environment_context(mapping.transformInboundPartition(new_part), environment_context);
  return mapping.transformOutboundPartition(result);
}
 
Example #14
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public org.apache.hadoop.hive.metastore.api.Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String srcDb,
    String srcTbl,
    String dstDb,
    String dstTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartition(partitionSpecs, srcDb, srcTbl, dstDb, dstTbl);
}
 
Example #15
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> exchange_partitions(
    Map<String, String> partitionSpecs,
    String sourceDb,
    String sourceTbl,
    String destDb,
    String destTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartitions(partitionSpecs, sourceDb, sourceTbl, destDb, destTbl);
}
 
Example #16
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public void createTableWithConstraints(
    org.apache.hadoop.hive.metastore.api.Table table,
    List<SQLPrimaryKey> primaryKeys,
    List<SQLForeignKey> foreignKeys
) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.createTableWithConstraints(table, primaryKeys, foreignKeys);
}
 
Example #17
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(request.getDbName());
  for (Partition partition : request.getParts()) {
    mapping.checkWritePermissions(partition.getDbName());
  }
  AddPartitionsResult result = mapping
      .getClient()
      .add_partitions_req(mapping.transformInboundAddPartitionsRequest(request));
  return mapping.transformOutboundAddPartitionsResult(result);
}
 
Example #18
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public boolean deleteTableColumnStatistics(
    String dbName, String tableName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deleteTableColumnStatistics(dbName, tableName, colName);
}
 
Example #19
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition(Partition new_part)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping.getClient().add_partition(mapping.transformInboundPartition(new_part));
  return mapping.transformOutboundPartition(result);
}
 
Example #20
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_constraints(Table tbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_constraints(mapping.transformInboundTable(tbl), primaryKeys, foreignKeys);
}
 
Example #21
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_with_environment_context(
    String db_name,
    String tbl_name,
    List<String> part_vals,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals,
          environment_context);
  return mapping.transformOutboundPartition(partition);
}
 
Example #22
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public boolean deletePartitionColumnStatistics(
    String dbName, String tableName, String partName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deletePartitionColumnStatistics(dbName, tableName, partName, colName);
}
 
Example #23
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public void createIndex(Index index, org.apache.hadoop.hive.metastore.api.Table indexTable) throws InvalidObjectException, MetaException, NoSuchObjectException,
      TException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException {
  boolean dirCreated = glueMetastoreClientDelegate.validateNewTableAndCreateDirectory(indexTable);
  boolean indexTableCreated = false;
  String dbName = index.getDbName();
  String indexTableName = index.getIndexTableName();
  String originTableName = index.getOrigTableName();
  Path indexTablePath = new Path(indexTable.getSd().getLocation());
  Table catalogIndexTableObject = HiveToCatalogConverter.convertIndexToTableObject(index);
  String indexTableObjectName = INDEX_PREFIX + index.getIndexName();

  try {
    org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, originTableName);
    Map<String, String> parameters = originTable.getParameters();
    if (parameters.containsKey(indexTableObjectName)){
      throw new org.apache.hadoop.hive.metastore.api.AlreadyExistsException("Index: " + index.getIndexName() + " already exist");
    }
    createTable(indexTable);
    indexTableCreated = true;
    originTable.getParameters().put(indexTableObjectName, catalogTableToString(catalogIndexTableObject));
    alter_table(dbName, originTableName, originTable);
  } catch (Exception e) {
    if (dirCreated){
      wh.deleteDir(indexTablePath, true);
    }
    if (indexTableCreated) {
      dropTable(dbName, indexTableName);
    }
    String msg = "Unable to create index: ";
    logger.error(msg, e);
    if (e instanceof TException) {
      throw e;
    } else {
      throw new MetaException(msg + e);
    }
  }
}
 
Example #24
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_table_column_statistics(String db_name, String tbl_name, String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_table_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 
Example #25
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions(List<Partition> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (Partition partition : new_parts) {
      mapping.checkWritePermissions(partition.getDbName());
    }
    return mapping.getClient().add_partitions(mapping.transformInboundPartitions(new_parts));
  }
  return 0;
}
 
Example #26
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public int add_partitions_pspec(
    PartitionSpecProxy pSpec
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException,
    MetaException, TException {
  return glueMetastoreClientDelegate.addPartitionsSpecProxy(pSpec);
}
 
Example #27
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index add_index(Index new_index, Table index_table)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_index.getDbName());
  mapping.checkWritePermissions(index_table.getDbName());
  Index result = mapping
      .getClient()
      .add_index(mapping.transformInboundIndex(new_index), mapping.transformInboundTable(index_table));
  return mapping.transformOutboundIndex(result);
}
 
Example #28
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name_with_environment_context(
    String db_name,
    String tbl_name,
    String part_name,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name,
          part_name, environment_context);
  return mapping.transformOutboundPartition(partition);
}
 
Example #29
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public org.apache.hadoop.hive.metastore.api.Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String srcDb,
    String srcTbl,
    String dstDb,
    String dstTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartition(partitionSpecs, srcDb, srcTbl, dstDb, dstTbl);
}
 
Example #30
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition(String db_name, String tbl_name, List<String> part_vals)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition result = mapping
      .getClient()
      .append_partition(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals);
  return mapping.transformOutboundPartition(result);
}