Java Code Examples for com.amazonaws.services.glue.model.Table

The following examples show how to use com.amazonaws.services.glue.model.Table. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testAlterIndex() throws Exception {
  Table catalogIndexTable = HiveToCatalogConverter.convertIndexToTableObject(testIndex);
  testTable.getParameters().put(INDEX_PREFIX + testIndex.getIndexName(), catalogTableToString(catalogIndexTable));

  when(glueClient.getTable(new GetTableRequest().withDatabaseName(testDB.getName()).withName(testTable.getTableName())))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getDatabase(new GetDatabaseRequest().withName(testDB.getName())))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));

  testIndex.setIndexHandlerClass("test_alter");
  Table updatedIndex = HiveToCatalogConverter.convertIndexToTableObject(testIndex);
  TableInput expectedTableInputWithIndex = GlueInputConverter.convertToTableInput(testTable);
  expectedTableInputWithIndex.getParameters().put(INDEX_PREFIX + testIndex.getIndexName(), catalogTableToString(updatedIndex));

  metastoreClient.alter_index(testDB.getName(), testTable.getTableName(), testIndex.getIndexName(), testIndex);

  // verify UpdateRequestTable call is made with expected table input containing new Index
  ArgumentCaptor<UpdateTableRequest> captor = ArgumentCaptor.forClass(UpdateTableRequest.class);
  verify(glueClient, times(1)).updateTable(captor.capture());
  assertEquals(expectedTableInputWithIndex, captor.getValue().getTableInput());
}
 
Example 2
@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForUnknownTable() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(sd);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);    
      
  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  doThrow(EntityNotFoundException.class).when(glueClient).getTable(any(GetTableRequest.class));
  
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(oldPartition));     
  
  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 
Example 3
/**
 * @return boolean
 *    true  -> directory created
 *    false -> directory not created
 */
public boolean validateNewTableAndCreateDirectory(org.apache.hadoop.hive.metastore.api.Table tbl) throws TException {
  checkNotNull(tbl, "tbl cannot be null");
  if (tableExists(tbl.getDbName(), tbl.getTableName())) {
    throw new AlreadyExistsException("Table " + tbl.getTableName() + " already exists.");
  }
  validateTableObject(tbl, conf);

  if (TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
    // we don't need to create directory for virtual views
    return false;
  }

  if (StringUtils.isEmpty(tbl.getSd().getLocation())) {
    org.apache.hadoop.hive.metastore.api.Database db = getDatabase(tbl.getDbName());
    tbl.getSd().setLocation(hiveShims.getDefaultTablePath(db, tbl.getTableName(), wh).toString());
  } else {
    tbl.getSd().setLocation(wh.getDnsPath(new Path(tbl.getSd().getLocation())).toString());
  }

  Path tblPath = new Path(tbl.getSd().getLocation());
  return makeDirs(wh, tblPath);
}
 
Example 4
/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
 
Example 5
public List<String> listPartitionNames(
    String databaseName,
    String tableName,
    List<String> values,
    short max
) throws TException {
  String expression = null;
  org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
  if (values != null) {
    expression = ExpressionHelper.buildExpressionFromPartialSpecification(table, values);
  }

  List<String> names = Lists.newArrayList();
  List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getPartitions(databaseName, tableName, expression, max);
  for(org.apache.hadoop.hive.metastore.api.Partition p : partitions) {
    names.add(Warehouse.makePartName(table.getPartitionKeys(), p.getValues()));
  }
  return names;
}
 
Example 6
public List<Index> listIndexes(String dbName, String tblName) throws TException {
  checkArgument(StringUtils.isNotEmpty(dbName), "dbName cannot be null or empty");
  checkArgument(StringUtils.isNotEmpty(tblName), "tblName cannot be null or empty");

  org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, tblName);
  Map<String, String> parameters = originTable.getParameters();
  List<Table> indexTableObjects = Lists.newArrayList();
  for(String key : parameters.keySet()) {
    if(key.startsWith(INDEX_PREFIX)) {
      String serialisedString = parameters.get(key);
      indexTableObjects.add(stringToCatalogTable(serialisedString));
    }
  }

  List<Index> hiveIndexList = Lists.newArrayList();
  for (Table catalogIndexTableObject : indexTableObjects) {
    hiveIndexList.add(CatalogToHiveConverter.convertTableObjectToIndex(catalogIndexTableObject));
  }
  return hiveIndexList;
}
 
Example 7
/**
 * Returns a list of tables from AWS Glue DataCatalog with optional filtering for the requested schema (aka database)
 *
 * @param blockAllocator Tool for creating and managing Apache Arrow Blocks.
 * @param request Provides details on who made the request and which Athena catalog they are querying.
 * @param filter The TableFilter to apply to all tables before adding them to the results list.
 * @return The ListTablesResponse which mostly contains the list of table names.
 */
protected ListTablesResponse doListTables(BlockAllocator blockAllocator, ListTablesRequest request, TableFilter filter)
        throws Exception
{
    GetTablesRequest getTablesRequest = new GetTablesRequest();
    getTablesRequest.setCatalogId(getCatalog(request));
    getTablesRequest.setDatabaseName(request.getSchemaName());

    Set<TableName> tables = new HashSet<>();
    String nextToken = null;
    do {
        getTablesRequest.setNextToken(nextToken);
        GetTablesResult result = awsGlue.getTables(getTablesRequest);

        for (Table next : result.getTableList()) {
            if (filter == null || filter.filter(next)) {
                tables.add(new TableName(request.getSchemaName(), next.getName()));
            }
        }

        nextToken = result.getNextToken();
    }
    while (nextToken != null);

    return new ListTablesResponse(request.getCatalogName(), tables);
}
 
Example 8
@Test
public void testConvertHiveTableToTableInput() {
  org.apache.hadoop.hive.metastore.api.Table hivetbl = CatalogToHiveConverter.convertTable(testTable, testDB.getName());
  TableInput tblInput = GlueInputConverter.convertToTableInput(hivetbl);

  assertEquals(testTable.getName(), tblInput.getName());
  assertEquals(testTable.getOwner(), tblInput.getOwner());
  assertEquals(testTable.getTableType(), tblInput.getTableType());
  assertEquals(testTable.getParameters(), tblInput.getParameters());
  assertEquals(testTable.getPartitionKeys(), tblInput.getPartitionKeys());
  assertEquals(testTable.getRetention(), tblInput.getRetention());
  assertEquals(testTable.getLastAccessTime(), tblInput.getLastAccessTime());
  assertEquals(testTable.getStorageDescriptor(), tblInput.getStorageDescriptor());
  assertEquals(testTable.getViewExpandedText(), tblInput.getViewExpandedText());
  assertEquals(testTable.getViewOriginalText(), tblInput.getViewOriginalText());
}
 
Example 9
@Test
public void testDropPartitionsEmpty() throws Exception {
  Table table = HiveToCatalogConverter.convertTable(testTable);

  String namespaceName = testDB.getName();
  String tableName = table.getName();

  List<String> values = Arrays.asList("foo", "bar");
  Partition partition = new Partition().withDatabaseName(namespaceName)
      .withTableName(tableName)
      .withValues(values)
      .withStorageDescriptor(TestObjects.getTestStorageDescriptor());

  mockGetPartitionsSuccess(Lists.<Partition>newArrayList());
  mockBatchDeleteSuccess();

  List<org.apache.hadoop.hive.metastore.api.Partition> partitions = metastoreClient.dropPartitions(
      namespaceName, tableName, Lists.newArrayList(getDumbExpression()), false, false, false);

  assertEquals(0, partitions.size());
  assertDaemonThreadPools();
}
 
Example 10
@Test
public void testUpdateTableWhenCacheEnabled() {
    TableInput tableInput = new TableInput();
    tableInput.setName(TABLE_NAME);
    AWSGlueMetastoreCacheDecorator cacheDecorator =
            new AWSGlueMetastoreCacheDecorator(hiveConf, glueMetastore);

    cacheDecorator.tableCache.put(TABLE_IDENTIFIER, new Table());
    doNothing().when(glueMetastore).updateTable(DB_NAME, tableInput);

    cacheDecorator.updateTable(DB_NAME, tableInput);

    //table should have been removed from cache
    assertNull(cacheDecorator.tableCache.getIfPresent(TABLE_IDENTIFIER));
    verify(glueMetastore, times(1)).updateTable(DB_NAME, tableInput);
}
 
Example 11
@Test
public void testGetTableMetaNullEmptyTableType() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = null;

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
    new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
    new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));

  tableTypes = Lists.newArrayList();
  tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 
Example 12
@Test
public void testAlterTable() throws Exception {
  org.apache.hadoop.hive.metastore.api.Table newHiveTable
      = CatalogToHiveConverter.convertTable(getTestTable(), testDb.getName());
  newHiveTable.setTableName(testTbl.getName());

  when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(new GetDatabaseResult().withDatabase((testDb)));
  when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable((testTbl)));
  metastoreClientDelegateCatalogId.alterTable(testDb.getName(), testTbl.getName(), newHiveTable, null);

  ArgumentCaptor<UpdateTableRequest> captor = ArgumentCaptor.forClass(UpdateTableRequest.class);
  verify(glueClient, times(1)).updateTable(captor.capture());

  TableInput expectedTableInput = GlueInputConverter.convertToTableInput(newHiveTable);
  assertEquals(expectedTableInput, captor.getValue().getTableInput());
}
 
Example 13
@Test
public void testValidateTableAndCreateDirectoryVirtualView() throws Exception {
  testTbl.setTableType(TableType.VIRTUAL_VIEW.toString());
  testTbl.getStorageDescriptor().setLocation(null);
  org.apache.hadoop.hive.metastore.api.Table hiveTbl = CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName());

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(testDb));
  when(glueClient.getTable(new GetTableRequest()
    .withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
    .thenThrow(EntityNotFoundException.class);

  assertFalse(metastoreClientDelegate.validateNewTableAndCreateDirectory(hiveTbl));
  assertNull(testTbl.getStorageDescriptor().getLocation());
  verify(wh, never()).mkdirs(any(Path.class), anyBoolean());
}
 
Example 14
@Test(expected = MetaException.class)
public void testAddPartitions_PartitionViewWithLocation() throws Exception {
  // Case: table location is empty (VIRTUAL_VIEW) with partition containing location
  // In Hive, this throws MetaException because it doesn't allow parititon views to have location
  Table table = testTbl;
  table.getStorageDescriptor().setLocation(null);

  int numPartitions = 2;
  List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);

  mockBatchCreatePartitionsSucceed();
  when(glueClient.getTable(any(GetTableRequest.class)))
    .thenReturn(new GetTableResult().withTable(table));
  when(wh.mkdirs(any(Path.class), anyBoolean())).thenReturn(true);

  metastoreClientDelegate.addPartitions(partitions, false, true);

  assertDaemonThreadPools();
}
 
Example 15
/**
 * Glue has strict table naming rules and may not be able to match the exact table name from the source. So this stores
 * the source table name in the schema metadata if necessary to ease lookup later. It looks for it in the following places:
 * <p><ul>
 * <li>A table property called {@value SOURCE_TABLE_PROPERTY}
 * <li>In StorageDescriptor.Location in the form of an ARN (e.g. arn:aws:dynamodb:us-east-1:012345678910:table/mytable)
 * </ul><p>
 * Override this method to fetch the source table name from somewhere else.
 *
 * @param table The Glue Table
 * @param schemaBuilder The schema being generated
 */
protected static void populateSourceTableNameIfAvailable(Table table, SchemaBuilder schemaBuilder)
{
    String sourceTableProperty = table.getParameters().get(SOURCE_TABLE_PROPERTY);
    if (sourceTableProperty != null) {
        // table property exists so nothing to do (assumes all table properties were already copied)
        return;
    }
    String location = table.getStorageDescriptor().getLocation();
    if (location != null) {
        Matcher matcher = TABLE_ARN_REGEX.matcher(location);
        if (matcher.matches()) {
            schemaBuilder.addMetadata(SOURCE_TABLE_PROPERTY, matcher.group(1));
        }
    }
}
 
Example 16
@Test
public void testDropPartitionUsingName() throws Exception {
  Table table = HiveToCatalogConverter.convertTable(testTable);
  List<String> values = Arrays.asList("foo", "bar");
  Partition partition = new Partition().withDatabaseName(testDB.getName())
      .withTableName(table.getName())
      .withValues(values)
      .withStorageDescriptor(TestObjects.getTestStorageDescriptor());
  when(glueClient.deletePartition(any(DeletePartitionRequest.class))).thenReturn(new DeletePartitionResult());
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(partition));
  when(glueClient.getTable(any(GetTableRequest.class)))
    .thenReturn(new GetTableResult().withTable(table));
  when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
    .thenReturn(new GetPartitionsResult().withPartitions(partition));
  // Warehouse is expecting a pattern of /key=val
  String partitionName = table.getPartitionKeys().get(0).getName() + "=foo";
  boolean deleteData = true;
  metastoreClient.dropPartition(testDB.getName(), table.getName(), partitionName, deleteData);
  verify(glueClient).deletePartition(any(DeletePartitionRequest.class));
  verify(glueClient).getPartition(any(GetPartitionRequest.class));
  verify(glueClient).getTable(any(GetTableRequest.class));
  assertDaemonThreadPools();
}
 
Example 17
private void performDropPartitionPostProcessing(String dbName, String tblName,
                                                org.apache.hadoop.hive.metastore.api.Partition partition, boolean deleteData, boolean ifPurge)
      throws MetaException, NoSuchObjectException, TException {
    if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
        Path partPath = new Path(partition.getSd().getLocation());
        org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
        if (isExternalTable(table)){
            //Don't delete external table data
            return;
        }
        boolean mustPurge = isMustPurge(table, ifPurge);
        wh.deleteDir(partPath, true, mustPurge);
        try {
            List<String> values = partition.getValues();
            deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
        } catch (IOException e) {
            throw new MetaException(e.getMessage());
        }
    }
}
 
Example 18
@Test
public void testDropIndex() throws Exception {
  Table catalogIndexTable = getTestTable();
  setIndexParametersForIndexTable(catalogIndexTable, testDB.getName(), testTable.getTableName());
  testIndex.setOrigTableName(testTable.getTableName());
  testIndex.setIndexTableName(catalogIndexTable.getName());
  testTable.getParameters().put(INDEX_PREFIX + testIndex.getIndexName(), catalogTableToString(catalogIndexTable));

  when(glueClient.getTable(new GetTableRequest().withDatabaseName(testDB.getName()).withName(testTable.getTableName())))
    .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));
  when(glueClient.getTable(new GetTableRequest().withDatabaseName(testDB.getName()).withName(testIndex.getIndexTableName())))
    .thenReturn(new GetTableResult().withTable(catalogIndexTable));
  when(glueClient.getDatabase(new GetDatabaseRequest().withName(testDB.getName())))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
    .thenReturn(new GetPartitionsResult().withPartitions(ImmutableList.<Partition>of()));

  metastoreClient.dropIndex(testDB.getName(), testTable.getTableName(), testIndex.getIndexName(), true);

  TableInput expectedTableInput = GlueInputConverter.convertToTableInput(testTable);
  verify(glueClient).updateTable(new UpdateTableRequest().withDatabaseName(testDB.getName()).withTableInput(expectedTableInput));
  verify(glueClient).deleteTable(new DeleteTableRequest().withDatabaseName(testDB.getName()).withName(testIndex.getIndexTableName()));
}
 
Example 19
@Override
public org.apache.hadoop.hive.metastore.api.Partition getPartitionWithAuthInfo(
      String databaseName, String tableName, List<String> values,
      String userName, List<String> groupNames)
      throws MetaException, UnknownTableException, NoSuchObjectException, TException {

    // TODO move this into the service
    org.apache.hadoop.hive.metastore.api.Partition partition = getPartition(databaseName, tableName, values);
    org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
    if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
        String partName = Warehouse.makePartName(table.getPartitionKeys(), values);
        HiveObjectRef obj = new HiveObjectRef();
        obj.setObjectType(HiveObjectType.PARTITION);
        obj.setDbName(databaseName);
        obj.setObjectName(tableName);
        obj.setPartValues(values);
        org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet =
              this.get_privilege_set(obj, userName, groupNames);
        partition.setPrivileges(privilegeSet);
    }

    return partition;
}
 
Example 20
@Override
public org.apache.hadoop.hive.metastore.api.Partition getPartitionWithAuthInfo(
      String databaseName, String tableName, List<String> values,
      String userName, List<String> groupNames)
      throws MetaException, UnknownTableException, NoSuchObjectException, TException {

  // TODO move this into the service
  org.apache.hadoop.hive.metastore.api.Partition partition = getPartition(databaseName, tableName, values);
  org.apache.hadoop.hive.metastore.api.Table table = getTable(databaseName, tableName);
  if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) {
    String partName = Warehouse.makePartName(table.getPartitionKeys(), values);
    HiveObjectRef obj = new HiveObjectRef();
    obj.setObjectType(HiveObjectType.PARTITION);
    obj.setDbName(databaseName);
    obj.setObjectName(tableName);
    obj.setPartValues(values);
    org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet =
          this.get_privilege_set(obj, userName, groupNames);
    partition.setPrivileges(privilegeSet);
  }

  return partition;
}
 
Example 21
private void verifyDestinationLocation(FileSystem srcFs, FileSystem destFs, Path srcPath, Path destPath, org.apache.hadoop.hive.metastore.api.Table tbl, org.apache.hadoop.hive.metastore.api.Partition newPartition)
      throws InvalidOperationException {
    String oldPartLoc = srcPath.toString();
    String newPartLoc = destPath.toString();

    // check that src and dest are on the same file system
    if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
        throw new InvalidOperationException("table new location " + destPath
              + " is on a different file system than the old location "
              + srcPath + ". This operation is not supported");
    }
    try {
        srcFs.exists(srcPath); // check that src exists and also checks
        if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
            throw new InvalidOperationException("New location for this partition "
                  + tbl.getDbName() + "." + tbl.getTableName() + "." + newPartition.getValues()
                  + " already exists : " + destPath);
        }
    } catch (IOException e) {
        throw new InvalidOperationException("Unable to access new location "
              + destPath + " for partition " + tbl.getDbName() + "."
              + tbl.getTableName() + " " + newPartition.getValues());
    }
}
 
Example 22
@Test(expected=InvalidOperationException.class)
public void testRenamePartitionForInvalidSD() throws Exception {
  String dbName = testDB.getName();
  Table externalTable = getTestTable();
  externalTable.setTableType(TableType.EXTERNAL_TABLE.name());   
  
  StorageDescriptor sd = HiveToCatalogConverter.convertStorageDescriptor(testPartition.getSd());
  
  Partition oldPartition = new Partition()
      .withDatabaseName(dbName).withTableName(externalTable.getName())
      .withValues(Lists.newArrayList("oldval")).withStorageDescriptor(null);
  
  Partition newPartition = new Partition()
                                    .withDatabaseName(dbName).withTableName(externalTable.getName())
                                    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(sd);

  when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
    .thenReturn(new GetDatabaseResult().withDatabase(HiveToCatalogConverter.convertDatabase(testDB)));
  when(glueClient.getTable(any(GetTableRequest.class)))
  .thenReturn(new GetTableResult().withTable(externalTable));
  when(glueClient.getPartition(any(GetPartitionRequest.class)))
    .thenReturn(new GetPartitionResult().withPartition(oldPartition));

  metastoreClient.renamePartition(dbName, externalTable.getName(), oldPartition.getValues(),
      CatalogToHiveConverter.convertPartition(newPartition));
}
 
Example 23
@Override
public void alter_index(String dbName, String tblName, String indexName, Index index) throws InvalidOperationException,
      MetaException, TException {
  Table catalogIndexTableObject = HiveToCatalogConverter.convertIndexToTableObject(index);
  org.apache.hadoop.hive.metastore.api.Table originTable = getTable(dbName, tblName);
  String indexTableObjectName = INDEX_PREFIX + indexName;
  if (!originTable.getParameters().containsKey(indexTableObjectName)) {
    throw new NoSuchObjectException("can not find index: " + indexName);
  }

  originTable.getParameters().put(indexTableObjectName, catalogTableToString(catalogIndexTableObject));
  alter_table(dbName, tblName, originTable);
}
 
Example 24
Source Project: presto   Source File: TestGlueToPrestoConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertTableNullPartitions()
{
    testTable.setPartitionKeys(null);
    io.prestosql.plugin.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(testTable, testDatabase.getName());
    assertTrue(prestoTable.getPartitionColumns().isEmpty());
}
 
Example 25
public org.apache.hadoop.hive.metastore.api.Partition appendPartition(
    String dbName,
    String tblName,
    List<String> values
) throws TException {
  checkArgument(StringUtils.isNotEmpty(dbName), "dbName cannot be null or empty");
  checkArgument(StringUtils.isNotEmpty(tblName), "tblName cannot be null or empty");
  checkNotNull(values, "partition values cannot be null");
  org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
  checkNotNull(table.getSd(), "StorageDescriptor cannot be null for Table " + tblName);
  org.apache.hadoop.hive.metastore.api.Partition partition = buildPartitionFromValues(table, values);
  addPartitions(Lists.newArrayList(partition), false, true);
  return partition;
}
 
Example 26
Source Project: presto   Source File: TestGlueToPrestoConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertTableWithoutTableType()
{
    Table table = getGlueTestTable(testDatabase.getName());
    table.setTableType(null);
    io.prestosql.plugin.hive.metastore.Table prestoTable = GlueToPrestoConverter.convertTable(table, testDatabase.getName());
    assertEquals(prestoTable.getTableType(), EXTERNAL_TABLE.name());
}
 
Example 27
@Test
public void testDropPartitionsPartialFailure() throws Exception {
  Table table = HiveToCatalogConverter.convertTable(testTable);
  String namespaceName = testDB.getName();
  String tableName = table.getName();
  List<String> values1 = Arrays.asList("foo1", "bar1");
  List<String> values2 = Arrays.asList("foo2", "bar2");
  Partition partition1 = getTestPartition(namespaceName, tableName, values1);
  Partition partition2 = getTestPartition(namespaceName, tableName, values2);
  List<Partition> partitions = Lists.newArrayList(partition1, partition2);

  mockGetPartitionsSuccess(partitions);
  mockBatchDeleteWithFailures(Lists.newArrayList(getPartitionError(values1,
      new EntityNotFoundException("EntityNotFoundException"))));
  when(glueClient.getTable(any(GetTableRequest.class)))
      .thenReturn(new GetTableResult().withTable(HiveToCatalogConverter.convertTable(testTable)));

  try {
    metastoreClient.dropPartitions(namespaceName, tableName,
        Lists.newArrayList(getDumbExpression()), true, false, false);
    fail("should throw");
  } catch (TException e) {
    verify(glueClient, times(1)).batchDeletePartition(any(BatchDeletePartitionRequest.class));
    verify(glueClient, never()).getPartition(any(GetPartitionRequest.class));
    verify(wh, times(1)).deleteDir(any(Path.class), eq(true), eq(false));
    assertThat(e, is(instanceOf(NoSuchObjectException.class)));
    assertThat(e.getMessage(), containsString("EntityNotFoundException"));
    assertDaemonThreadPools();
  }
}
 
Example 28
@Test
public void testExpressionConversionWithMultipleNots() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100" ).val("val1").strCol("strCol").val(true).pred("between", 4)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .pred("and", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol) NOT IN ('val3', 'val2', 'val1')) and strCol NOT BETWEEN 'val1' AND 'val100')", filter);
}
 
Example 29
@Override
public List<Table> getTables(String dbname, String tablePattern) {
    List<Table> ret = new ArrayList<>();
    String nextToken = null;
    do {
        GetTablesRequest getTablesRequest = new GetTablesRequest().withDatabaseName(dbname)
                .withExpression(tablePattern).withNextToken(nextToken).withCatalogId(catalogId);
        GetTablesResult result = glueClient.getTables(getTablesRequest);
        ret.addAll(result.getTableList());
        nextToken = result.getNextToken();
    } while (nextToken != null);
    return ret;
}
 
Example 30
@Test
public void testTableConversionWithNullParameterMap() {
  // Test to ensure the parameter map returned to Hive is never null.
  Table catalogTable = TestObjects.getTestTable();
  catalogTable.setParameters(null);
  org.apache.hadoop.hive.metastore.api.Table hiveTable = CatalogToHiveConverter.convertTable(catalogTable, TEST_DB_NAME);
  assertNotNull(hiveTable.getParameters());
  assertTrue(hiveTable.getParameters().isEmpty());
}