Java Code Examples for org.apache.hadoop.hive.metastore.api.InvalidObjectException

The following examples show how to use org.apache.hadoop.hive.metastore.api.InvalidObjectException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Taken from HiveMetaStore#create_table_core
 * https://github.com/apache/hive/blob/rel/release-2.3.0/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java#L1370-L1383
 */
public static void validateTableObject(Table table, Configuration conf) throws InvalidObjectException {
  checkNotNull(table, "table cannot be null");
  checkNotNull(table.getSd(), "Table#StorageDescriptor cannot be null");

  if (!hiveShims.validateTableName(table.getTableName(), conf)) {
    throw new InvalidObjectException(table.getTableName() + " is not a valid object name");
  }
  String validate = MetaStoreUtils.validateTblColumns(table.getSd().getCols());
  if (validate != null) {
    throw new InvalidObjectException("Invalid column " + validate);
  }

  if (table.getPartitionKeys() != null) {
    validate = MetaStoreUtils.validateTblColumns(table.getPartitionKeys());
    if (validate != null) {
      throw new InvalidObjectException("Invalid partition column " + validate);
    }
  }
}
 
Example 2
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String source_db,
    String source_table_name,
    String dest_db,
    String dest_table_name)
    throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(source_db);
  mapping.checkWritePermissions(dest_db);
  Partition result = mapping
      .getClient()
      .exchange_partition(partitionSpecs, mapping.transformInboundDatabaseName(source_db), source_table_name,
          mapping.transformInboundDatabaseName(dest_db), dest_table_name);
  return mapping.transformOutboundPartition(result);
}
 
Example 3
@Test
public void testDeletePartitionsThrowsInvalidInputException() throws Exception {
  Exception e = new com.amazonaws.services.glue.model.InvalidInputException("foo");
  mockBatchDeleteThrowsException(e);

  List<String> values1 = Lists.newArrayList("val1");
  List<String> values2 = Lists.newArrayList("val2");
  List<Partition> partitions = Lists.newArrayList(
        TestObjects.getTestPartition(NAMESPACE_NAME, TABLE_NAME, values1),
        TestObjects.getTestPartition(NAMESPACE_NAME, TABLE_NAME, values2));
  batchDeletePartitionsHelper = new BatchDeletePartitionsHelper(client, NAMESPACE_NAME, TABLE_NAME, null, partitions);

  batchDeletePartitionsHelper.deletePartitions();
  assertTrue(batchDeletePartitionsHelper.getPartitionsDeleted().isEmpty());
  assertThat(batchDeletePartitionsHelper.getFirstTException(), is(instanceOf(InvalidObjectException.class)));
}
 
Example 4
@Test
public void testAddPartitionsThrowsExceptionSecondPage() throws Exception {
  int numPartitions = 200;
  int secondPageSize = numPartitions - BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE;
  when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
    .thenReturn(new BatchCreatePartitionResult())
    .thenThrow(new InvalidInputException("exception"));
  when(glueClient.getTable(any(GetTableRequest.class)))
    .thenReturn(new GetTableResult().withTable(testTbl));

  List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);

  try {
    metastoreClientDelegate.addPartitions(partitions, false, true);
    fail("Should throw");
  } catch (Exception e) {
    assertThat(e, is(instanceOf(InvalidObjectException.class)));
    verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
    verify(glueClient, times(2)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
    verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
    verify(wh, times(secondPageSize)).deleteDir(any(Path.class), eq(true));
    assertDaemonThreadPools();
  }
}
 
Example 5
@Test
public void testDropPartitionsClientException() throws Exception {
  Table table = HiveToCatalogConverter.convertTable(testTable);
  String namespaceName = testDB.getName();
  String tableName = table.getName();
  Partition partition = TestObjects.getTestPartition(namespaceName, tableName, Arrays.asList("foo", "bar"));

  mockGetPartitionsSuccess(Lists.newArrayList(partition));
  mockBatchDeleteThrowsException(new InvalidInputException("InvalidInputException"));
  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));

  try {
    metastoreClient.dropPartitions(namespaceName, tableName,
        Lists.newArrayList(getDumbExpression()), true, false, false);
    fail("should throw");
  } catch (TException e) {
    verify(glueClient, times(1)).batchDeletePartition(any(BatchDeletePartitionRequest.class));
    verify(glueClient, never()).getPartition(any(GetPartitionRequest.class));
    verify(wh, never()).deleteDir(any(Path.class), anyBoolean(), anyBoolean());
    assertThat(e, is(instanceOf(InvalidObjectException.class)));
    assertThat(e.getMessage(), containsString("InvalidInputException"));
  }
  assertDaemonThreadPools();
}
 
Example 6
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<Partition> exchange_partitions(
    Map<String, String> partitionSpecs,
    String source_db,
    String source_table_name,
    String dest_db,
    String dest_table_name)
    throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(source_db);
  mapping.checkWritePermissions(dest_db);
  List<Partition> result = mapping
      .getClient()
      .exchange_partitions(partitionSpecs, mapping.transformInboundDatabaseName(source_db), source_table_name,
          mapping.transformInboundDatabaseName(dest_db), dest_table_name);
  return mapping.transformOutboundPartitions(result);
}
 
Example 7
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name_with_environment_context(
    String db_name,
    String tbl_name,
    String part_name,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name,
          part_name, environment_context);
  return mapping.transformOutboundPartition(partition);
}
 
Example 8
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean delete_table_column_statistics(String db_name, String tbl_name, String col_name)
    throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  return mapping
      .getClient()
      .delete_table_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, col_name);
}
 
Example 9
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean update_table_column_statistics(ColumnStatistics stats_obj)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  DatabaseMapping mapping = checkWritePermissions(stats_obj.getStatsDesc().getDbName());
  return mapping.getClient().update_table_column_statistics(mapping.transformInboundColumnStatistics(stats_obj));
}
 
Example 10
@Override
public org.apache.hadoop.hive.metastore.api.Partition add_partition(org.apache.hadoop.hive.metastore.api.Partition partition)
      throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException, MetaException,
      TException {
  glueMetastoreClientDelegate.addPartitions(Lists.newArrayList(partition), false, true);
  return partition;
}
 
Example 11
@Override
public int add_partitions_pspec(
    PartitionSpecProxy pSpec
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException,
    MetaException, TException {
  return glueMetastoreClientDelegate.addPartitionsSpecProxy(pSpec);
}
 
Example 12
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_partition_column_statistics(
    String db_name,
    String tbl_name,
    String part_name,
    String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_partition_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, part_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 
Example 13
@Override
public void createIndex(Index index, org.apache.hadoop.hive.metastore.api.Table indexTable) throws InvalidObjectException, MetaException, NoSuchObjectException,
      TException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException {
    boolean dirCreated = glueMetastoreClientDelegate.validateNewTableAndCreateDirectory(indexTable);
    boolean indexTableCreated = false;
    String dbName = index.getDbName();
    String indexTableName = index.getIndexTableName();
    String originTableName = index.getOrigTableName();
    Path indexTablePath = new Path(indexTable.getSd().getLocation());
    Table catalogIndexTableObject = HiveToCatalogConverter.convertIndexToTableObject(index);
    String indexTableObjectName = INDEX_PREFIX + index.getIndexName();

    try {
        org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, originTableName);
        Map<String, String> parameters = originTable.getParameters();
        if (parameters.containsKey(indexTableObjectName)){
            throw new org.apache.hadoop.hive.metastore.api.AlreadyExistsException("Index: " + index.getIndexName() + " already exist");
        }
        createTable(indexTable);
        indexTableCreated = true;
        originTable.getParameters().put(indexTableObjectName, catalogTableToString(catalogIndexTableObject));
        alter_table(dbName, originTableName, originTable);
    } catch (Exception e) {
        if (dirCreated){
            wh.deleteDir(indexTablePath, true);
        }
        if (indexTableCreated) {
            dropTable(dbName, indexTableName);
        }
        String msg = "Unable to create index: ";
        logger.error(msg, e);
        if (e instanceof TException) {
            throw e;
        } else {
            throw new MetaException(msg + e);
        }
    }
}
 
Example 14
@Override
public boolean deletePartitionColumnStatistics(
    String dbName, String tableName, String partName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deletePartitionColumnStatistics(dbName, tableName, partName, colName);
}
 
Example 15
@Override
public boolean deleteTableColumnStatistics(
    String dbName, String tableName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deleteTableColumnStatistics(dbName, tableName, colName);
}
 
Example 16
@Override
public org.apache.hadoop.hive.metastore.api.Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String srcDb,
    String srcTbl,
    String dstDb,
    String dstTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartition(partitionSpecs, srcDb, srcTbl, dstDb, dstTbl);
}
 
Example 17
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition(String db_name, String tbl_name, List<String> part_vals)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition result = mapping
      .getClient()
      .append_partition(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals);
  return mapping.transformOutboundPartition(result);
}
 
Example 18
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index add_index(Index new_index, Table index_table)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_index.getDbName());
  mapping.checkWritePermissions(index_table.getDbName());
  Index result = mapping
      .getClient()
      .add_index(mapping.transformInboundIndex(new_index), mapping.transformInboundTable(index_table));
  return mapping.transformOutboundIndex(result);
}
 
Example 19
@Override
public int add_partitions_pspec(
    PartitionSpecProxy pSpec
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException,
    MetaException, TException {
  return glueMetastoreClientDelegate.addPartitionsSpecProxy(pSpec);
}
 
Example 20
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions(List<Partition> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (Partition partition : new_parts) {
      mapping.checkWritePermissions(partition.getDbName());
    }
    return mapping.getClient().add_partitions(mapping.transformInboundPartitions(new_parts));
  }
  return 0;
}
 
Example 21
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public ColumnStatistics get_table_column_statistics(String db_name, String tbl_name, String col_name)
    throws NoSuchObjectException, MetaException, InvalidInputException, InvalidObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  ColumnStatistics result = mapping
      .getClient()
      .get_table_column_statistics(mapping.transformInboundDatabaseName(db_name), tbl_name, col_name);
  return mapping.transformOutboundColumnStatistics(result);
}
 
Example 22
@Override
public void createIndex(Index index, org.apache.hadoop.hive.metastore.api.Table indexTable) throws InvalidObjectException, MetaException, NoSuchObjectException,
      TException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException {
  boolean dirCreated = glueMetastoreClientDelegate.validateNewTableAndCreateDirectory(indexTable);
  boolean indexTableCreated = false;
  String dbName = index.getDbName();
  String indexTableName = index.getIndexTableName();
  String originTableName = index.getOrigTableName();
  Path indexTablePath = new Path(indexTable.getSd().getLocation());
  Table catalogIndexTableObject = HiveToCatalogConverter.convertIndexToTableObject(index);
  String indexTableObjectName = INDEX_PREFIX + index.getIndexName();

  try {
    org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, originTableName);
    Map<String, String> parameters = originTable.getParameters();
    if (parameters.containsKey(indexTableObjectName)){
      throw new org.apache.hadoop.hive.metastore.api.AlreadyExistsException("Index: " + index.getIndexName() + " already exist");
    }
    createTable(indexTable);
    indexTableCreated = true;
    originTable.getParameters().put(indexTableObjectName, catalogTableToString(catalogIndexTableObject));
    alter_table(dbName, originTableName, originTable);
  } catch (Exception e) {
    if (dirCreated){
      wh.deleteDir(indexTablePath, true);
    }
    if (indexTableCreated) {
      dropTable(dbName, indexTableName);
    }
    String msg = "Unable to create index: ";
    logger.error(msg, e);
    if (e instanceof TException) {
      throw e;
    } else {
      throw new MetaException(msg + e);
    }
  }
}
 
Example 23
@Override
public boolean deletePartitionColumnStatistics(
    String dbName, String tableName, String partName, String colName
) throws NoSuchObjectException, MetaException, InvalidObjectException,
    TException, org.apache.hadoop.hive.metastore.api.InvalidInputException {
  return glueMetastoreClientDelegate.deletePartitionColumnStatistics(dbName, tableName, partName, colName);
}
 
Example 24
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_with_environment_context(
    String db_name,
    String tbl_name,
    List<String> part_vals,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals,
          environment_context);
  return mapping.transformOutboundPartition(partition);
}
 
Example 25
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(request.getDbName());
  for (Partition partition : request.getParts()) {
    mapping.checkWritePermissions(partition.getDbName());
  }
  AddPartitionsResult result = mapping
      .getClient()
      .add_partitions_req(mapping.transformInboundAddPartitionsRequest(request));
  return mapping.transformOutboundAddPartitionsResult(result);
}
 
Example 26
@Override
public org.apache.hadoop.hive.metastore.api.Partition exchange_partition(
    Map<String, String> partitionSpecs,
    String srcDb,
    String srcTbl,
    String dstDb,
    String dstTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartition(partitionSpecs, srcDb, srcTbl, dstDb, dstTbl);
}
 
Example 27
@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> exchange_partitions(
    Map<String, String> partitionSpecs,
    String sourceDb,
    String sourceTbl,
    String destDb,
    String destTbl
) throws MetaException, NoSuchObjectException, InvalidObjectException, TException {
  return glueMetastoreClientDelegate.exchangePartitions(partitionSpecs, sourceDb, sourceTbl, destDb, destTbl);
}
 
Example 28
@Override
public void createTableWithConstraints(
    org.apache.hadoop.hive.metastore.api.Table table,
    List<SQLPrimaryKey> primaryKeys,
    List<SQLForeignKey> foreignKeys
) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.createTableWithConstraints(table, primaryKeys, foreignKeys);
}
 
Example 29
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition_with_environment_context(Partition new_part, EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping
      .getClient()
      .add_partition_with_environment_context(mapping.transformInboundPartition(new_part), environment_context);
  return mapping.transformOutboundPartition(result);
}
 
Example 30
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}