package com.amazonaws.glue.catalog.metastore;
import com.amazonaws.glue.catalog.converters.CatalogToHiveConverter;
import com.amazonaws.glue.catalog.converters.GlueInputConverter;
import com.amazonaws.glue.catalog.util.TestObjects;
import com.amazonaws.glue.catalog.util.TestExecutorServiceFactory;
import com.amazonaws.services.glue.AWSGlue;
import com.amazonaws.services.glue.model.BatchCreatePartitionRequest;
import com.amazonaws.services.glue.model.BatchCreatePartitionResult;
import com.amazonaws.services.glue.model.BatchGetPartitionRequest;
import com.amazonaws.services.glue.model.BatchGetPartitionResult;
import com.amazonaws.services.glue.model.CreateDatabaseRequest;
import com.amazonaws.services.glue.model.CreateTableRequest;
import com.amazonaws.services.glue.model.CreateUserDefinedFunctionRequest;
import com.amazonaws.services.glue.model.Database;
import com.amazonaws.services.glue.model.DeleteDatabaseRequest;
import com.amazonaws.services.glue.model.DeletePartitionRequest;
import com.amazonaws.services.glue.model.DeletePartitionResult;
import com.amazonaws.services.glue.model.DeleteTableRequest;
import com.amazonaws.services.glue.model.DeleteUserDefinedFunctionRequest;
import com.amazonaws.services.glue.model.EntityNotFoundException;
import com.amazonaws.services.glue.model.GetDatabaseRequest;
import com.amazonaws.services.glue.model.GetDatabaseResult;
import com.amazonaws.services.glue.model.GetDatabasesRequest;
import com.amazonaws.services.glue.model.GetDatabasesResult;
import com.amazonaws.services.glue.model.GetPartitionRequest;
import com.amazonaws.services.glue.model.GetPartitionResult;
import com.amazonaws.services.glue.model.GetPartitionsRequest;
import com.amazonaws.services.glue.model.GetPartitionsResult;
import com.amazonaws.services.glue.model.GetTableRequest;
import com.amazonaws.services.glue.model.GetTableResult;
import com.amazonaws.services.glue.model.GetTablesRequest;
import com.amazonaws.services.glue.model.GetTablesResult;
import com.amazonaws.services.glue.model.GetUserDefinedFunctionRequest;
import com.amazonaws.services.glue.model.GetUserDefinedFunctionResult;
import com.amazonaws.services.glue.model.GetUserDefinedFunctionsRequest;
import com.amazonaws.services.glue.model.GetUserDefinedFunctionsResult;
import com.amazonaws.services.glue.model.InternalServiceException;
import com.amazonaws.services.glue.model.InvalidInputException;
import com.amazonaws.services.glue.model.OperationTimeoutException;
import com.amazonaws.services.glue.model.Partition;
import com.amazonaws.services.glue.model.PartitionInput;
import com.amazonaws.services.glue.model.Table;
import com.amazonaws.services.glue.model.TableInput;
import com.amazonaws.services.glue.model.UpdateDatabaseRequest;
import com.amazonaws.services.glue.model.UpdatePartitionRequest;
import com.amazonaws.services.glue.model.UpdatePartitionResult;
import com.amazonaws.services.glue.model.UpdateTableRequest;
import com.amazonaws.services.glue.model.UpdateUserDefinedFunctionRequest;
import com.amazonaws.services.glue.model.UserDefinedFunction;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.PrincipalType;
import org.apache.hadoop.hive.metastore.api.TableMeta;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Set;
import static com.amazonaws.glue.catalog.util.TestObjects.getTestDatabase;
import static com.amazonaws.glue.catalog.util.TestObjects.getTestPartition;
import static com.amazonaws.glue.catalog.util.TestObjects.getTestTable;
import static org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE;
import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.empty;
import static org.hamcrest.Matchers.instanceOf;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.not;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class GlueMetastoreClientDelegateTest {
private GlueMetastoreClientDelegate metastoreClientDelegate;
private GlueMetastoreClientDelegate metastoreClientDelegateCatalogId;
private HiveConf conf;
HiveConf hiveConfCatalogId; // conf with CatalogId
private AWSGlue glueClient;
private Warehouse wh;
private Database testDb;
private Table testTbl;
private static final int BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE = 100;
private static final int BATCH_GET_PARTITIONS_MAX_REQUEST_SIZE = 1000;
private static final String CATALOG_ID = "12345";
@Before
public void setup() throws Exception {
conf = new HiveConf();
glueClient = mock(AWSGlue.class);
wh = mock(Warehouse.class);
metastoreClientDelegate = new GlueMetastoreClientDelegate(conf, new DefaultAWSGlueMetastore(conf, glueClient), wh);
// Create a client delegate with CatalogId
hiveConfCatalogId = new HiveConf();
hiveConfCatalogId.set(GlueMetastoreClientDelegate.CATALOG_ID_CONF, CATALOG_ID);
metastoreClientDelegateCatalogId = new GlueMetastoreClientDelegate(hiveConfCatalogId, new DefaultAWSGlueMetastore(hiveConfCatalogId, glueClient), wh);
testDb = getTestDatabase();
testTbl= getTestTable(testDb.getName());
setupMockWarehouseForPath(new Path(testTbl.getStorageDescriptor().getLocation().toString()), false, true);
}
private void setupMockWarehouseForPath(Path path, boolean isDir, boolean mkDir) throws Exception {
when(wh.getDnsPath(path)).thenReturn(path);
when(wh.isDir(path)).thenReturn(isDir);
when(wh.mkdirs(path, true)).thenReturn(mkDir);
}
// ===================== Thread Executor =====================
@Test
public void testExecutorService() throws Exception {
Object defaultExecutorService = new DefaultExecutorServiceFactory().getExecutorService(conf);
assertEquals("Default executor service should be used", metastoreClientDelegate.getExecutorService(), defaultExecutorService);
HiveConf customConf = new HiveConf();
customConf.set(GlueMetastoreClientDelegate.CATALOG_ID_CONF, CATALOG_ID);
customConf.setClass(GlueMetastoreClientDelegate.CUSTOM_EXECUTOR_FACTORY_CONF, TestExecutorServiceFactory.class, ExecutorServiceFactory.class);
GlueMetastoreClientDelegate customDelegate = new GlueMetastoreClientDelegate(customConf, mock(AWSGlueMetastore.class), mock(Warehouse.class));
Object customExecutorService = new TestExecutorServiceFactory().getExecutorService(customConf);
assertEquals("Custom executor service should be used", customDelegate.getExecutorService(), customExecutorService);
}
// ===================== Database =====================
@Test
public void testCreateDatabaseWithExistingDir() throws Exception {
Path dbPath = new Path(testDb.getLocationUri());
setupMockWarehouseForPath(dbPath, true, true);
metastoreClientDelegate.createDatabase(CatalogToHiveConverter.convertDatabase(testDb));
verify(glueClient, times(1)).createDatabase(any(CreateDatabaseRequest.class));
verify(wh, times(1)).isDir(dbPath);
verify(wh, never()).mkdirs(dbPath, true);
}
@Test
public void testCreateDatabaseWithExistingDirWthCatalogId() throws Exception {
Path dbPath = new Path(testDb.getLocationUri());
setupMockWarehouseForPath(dbPath, true, true);
metastoreClientDelegateCatalogId.createDatabase(CatalogToHiveConverter.convertDatabase(testDb));
ArgumentCaptor<CreateDatabaseRequest> captor = ArgumentCaptor.forClass(CreateDatabaseRequest.class);
verify(glueClient, times(1)).createDatabase(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
verify(wh, times(1)).isDir(dbPath);
verify(wh, never()).mkdirs(dbPath, true);
}
@Test
public void testCreateDatabaseWithoutExistingDir() throws Exception {
Path dbPath = new Path(testDb.getLocationUri());
setupMockWarehouseForPath(dbPath, false, true);
metastoreClientDelegate.createDatabase(CatalogToHiveConverter.convertDatabase(testDb));
verify(glueClient, times(1)).createDatabase(any(CreateDatabaseRequest.class));
verify(wh, times(1)).isDir(dbPath);
verify(wh, times(1)).mkdirs(dbPath, true);
}
@Test
public void testGetDatabases() throws Exception {
when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
new GetDatabasesResult().withDatabaseList(testDb));
List<String> dbs = metastoreClientDelegate.getDatabases("*");
assertEquals(testDb.getName(), Iterables.getOnlyElement(dbs));
}
@Test
public void testGetDatabasesWithCatalogId() throws Exception {
when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
new GetDatabasesResult().withDatabaseList(testDb));
List<String> dbs = metastoreClientDelegateCatalogId.getDatabases("*");
ArgumentCaptor<GetDatabasesRequest> captor = ArgumentCaptor.forClass(GetDatabasesRequest.class);
verify(glueClient, times(1)).getDatabases(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
assertEquals(testDb.getName(), Iterables.getOnlyElement(dbs));
}
@Test
public void testGetDatabasesNullPattern() throws Exception {
when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
new GetDatabasesResult().withDatabaseList(testDb));
List<String> dbs = metastoreClientDelegate.getDatabases(null);
assertEquals(testDb.getName(), Iterables.getOnlyElement(dbs));
}
@Test
public void testGetDatabase() throws Exception {
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(
new GetDatabaseResult().withDatabase(getTestDatabase()));
metastoreClientDelegate.getDatabase("db");
verify(glueClient, atLeastOnce()).getDatabase(any(GetDatabaseRequest.class));
}
@Test
public void testGetDatabaseWithCatalogId() throws Exception {
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(
new GetDatabaseResult().withDatabase(getTestDatabase()));
metastoreClientDelegateCatalogId.getDatabase("db");
ArgumentCaptor<GetDatabaseRequest> captor = ArgumentCaptor.forClass(GetDatabaseRequest.class);
verify(glueClient, atLeastOnce()).getDatabase(captor.capture());
GetDatabaseRequest request = captor.getValue();
assertEquals(CATALOG_ID, request.getCatalogId());
assertEquals("db", request.getName());
}
@Test
public void testGetAllDatabases() throws Exception {
when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
new GetDatabasesResult().withDatabaseList(getTestDatabase()));
metastoreClientDelegate.getDatabases("*");
// Ensure this gets invoked
verify(glueClient, atLeastOnce()).getDatabases(any(GetDatabasesRequest.class));
}
@Test
public void testGetAllDatabasesPaginated() throws Exception {
when(glueClient.getDatabases(any(GetDatabasesRequest.class)))
.thenReturn(new GetDatabasesResult().withDatabaseList(testDb).withNextToken("token"))
.thenReturn(new GetDatabasesResult().withDatabaseList(getTestDatabase()));
List<String> databases = metastoreClientDelegate.getDatabases(".*");
assertEquals(2, databases.size());
verify(glueClient, times(2)).getDatabases(any(GetDatabasesRequest.class));
}
@Test
public void testAlterDatabase() throws Exception {
metastoreClientDelegate.alterDatabase("db", CatalogToHiveConverter.convertDatabase(testDb));
verify(glueClient, times(1)).updateDatabase(any(UpdateDatabaseRequest.class));
}
@Test
public void testAlterDatabaseWithCatalogId() throws Exception {
metastoreClientDelegateCatalogId.alterDatabase("db", CatalogToHiveConverter.convertDatabase(testDb));
ArgumentCaptor<UpdateDatabaseRequest> captor = ArgumentCaptor.forClass(UpdateDatabaseRequest.class);
verify(glueClient, times(1)).updateDatabase(any(UpdateDatabaseRequest.class));
verify(glueClient).updateDatabase(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testDropDatabaseDeleteData() throws Exception {
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(
new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
new GetTablesResult().withTableList(ImmutableList.<Table>of()));
Path dbPath = new Path(testDb.getLocationUri());
when(wh.deleteDir(dbPath, true)).thenReturn(true);
metastoreClientDelegate.dropDatabase(testDb.getName(), true, false, false);
verify(glueClient, times(1)).deleteDatabase(any(DeleteDatabaseRequest.class));
verify(wh, times(1)).deleteDir(dbPath, true);
}
@Test
public void testDropDatabaseDeleteDataWithCatalogId() throws Exception {
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(
new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
new GetTablesResult().withTableList(ImmutableList.<Table>of()));
Path dbPath = new Path(testDb.getLocationUri());
when(wh.deleteDir(dbPath, true)).thenReturn(true);
metastoreClientDelegateCatalogId.dropDatabase(testDb.getName(), true, false, false);
ArgumentCaptor<DeleteDatabaseRequest> captor = ArgumentCaptor.forClass(DeleteDatabaseRequest.class);
verify(glueClient, times(1)).deleteDatabase(captor.capture());
DeleteDatabaseRequest request = captor.getValue();
verify(wh, times(1)).deleteDir(dbPath, true);
assertEquals(CATALOG_ID, request.getCatalogId());
assertEquals(testDb.getName(), request.getName());
}
@Test
public void testDropDatabaseKeepData() throws Exception {
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(
new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
new GetTablesResult().withTableList(ImmutableList.<Table>of()));
Path dbPath = new Path(testDb.getLocationUri());
when(wh.deleteDir(dbPath, true)).thenReturn(true);
metastoreClientDelegate.dropDatabase(testDb.getName(), false, false, false);
verify(glueClient, times(1)).deleteDatabase(any(DeleteDatabaseRequest.class));
verify(wh, never()).deleteDir(dbPath, true);
}
// ======================= Table ======================
@Test(expected = InvalidObjectException.class)
public void testGetTableInvalidGlueTable() throws Exception {
Table tbl = getTestTable().withTableType(null);
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(tbl));
metastoreClientDelegate.getTable(testDb.getName(), tbl.getName());
}
@Test
public void testGetTables() throws Exception {
Table tbl2 = getTestTable();
List<String> tableNames = ImmutableList.of(testTbl.getName(), tbl2.getName());
List<Table> tableList = ImmutableList.of(testTbl, tbl2);
when(glueClient.getTables(new GetTablesRequest().withDatabaseName(testDb.getName()).withExpression("*")))
.thenReturn(new GetTablesResult().withTableList(tableList));
List<String> result = metastoreClientDelegate.getTables(testDb.getName(), "*");
verify(glueClient).getTables(new GetTablesRequest().withDatabaseName(testDb.getName()).withExpression("*"));
assertThat(result, is(tableNames));
}
@Test
public void testGetTableWithCatalogId() throws Exception {
Table tbl2 = getTestTable();
List<String> tableNames = ImmutableList.of(testTbl.getName(), tbl2.getName());
List<Table> tableList = ImmutableList.of(testTbl, tbl2);
when(glueClient.getTables(new GetTablesRequest().withDatabaseName(testDb.getName()).withExpression("*").withCatalogId(CATALOG_ID)))
.thenReturn(new GetTablesResult().withTableList(tableList));
List<String> result = metastoreClientDelegateCatalogId.getTables(testDb.getName(), "*");
assertThat(result, is(tableNames));
ArgumentCaptor<GetTablesRequest> captor = ArgumentCaptor.forClass(GetTablesRequest.class);
verify(glueClient, times(1)).getTables(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
assertEquals(testDb.getName(), captor.getValue().getDatabaseName());
assertEquals("*", captor.getValue().getExpression());
}
@Test
public void testGetTablesWithPagination() throws Exception {
Table tbl2 = getTestTable();
List<String> tableNames = ImmutableList.of(testTbl.getName(), tbl2.getName());
List<Table> tableList1 = ImmutableList.of(testTbl);
List<Table> tableList2 = ImmutableList.of(tbl2);
String nextToken = "1";
when(glueClient.getTables(any(GetTablesRequest.class)))
.thenReturn(new GetTablesResult().withTableList(tableList1).withNextToken(nextToken))
.thenReturn(new GetTablesResult().withTableList(tableList2));
List<String> result = metastoreClientDelegate.getTables(testDb.getName(), "*");
verify(glueClient, times(2)).getTables(any(GetTablesRequest.class));
assertThat(result, is(tableNames));
}
@Test
public void testGetTableMeta() throws Exception {
List<Table> tables = Lists.newArrayList(testTbl);
List<String> tableTypes = Lists.newArrayList(TableType.MANAGED_TABLE.name());
when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
new GetDatabasesResult().withDatabaseList(testDb));
when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
new GetTablesResult().withTableList(tables));
List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
@Test
public void testGetTableMetaNullEmptyTableType() throws Exception {
List<Table> tables = Lists.newArrayList(testTbl);
List<String> tableTypes = null;
when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
new GetDatabasesResult().withDatabaseList(testDb));
when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
new GetTablesResult().withTableList(tables));
List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
tableTypes = Lists.newArrayList();
tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
@Test
public void testCreateTableWithExistingDir() throws Exception {
Path tblPath = new Path(testTbl.getStorageDescriptor().getLocation());
setupMockWarehouseForPath(tblPath, true, true);
when(glueClient.getDatabase(new GetDatabaseRequest().withName(testDb.getName())))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTable(new GetTableRequest().withDatabaseName(testTbl.getDatabaseName())
.withName(testTbl.getName()))).thenThrow(new EntityNotFoundException(""));
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
metastoreClientDelegate.createTable(CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName()));
verify(glueClient, times(1)).createTable(any(CreateTableRequest.class));
verify(wh).isDir(tblPath);
verify(wh, never()).mkdirs(tblPath, true);
}
@Test
public void testCreateTableWithExistingDirWithCatalogId() throws Exception {
Path tblPath = new Path(testTbl.getStorageDescriptor().getLocation());
setupMockWarehouseForPath(tblPath, true, true);
when(glueClient.getDatabase(new GetDatabaseRequest().withName(testDb.getName()).withCatalogId(CATALOG_ID)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTable(new GetTableRequest().withDatabaseName(testTbl.getDatabaseName()).withCatalogId(CATALOG_ID)
.withName(testTbl.getName()))).thenThrow(new EntityNotFoundException(""));
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
metastoreClientDelegateCatalogId.createTable(CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName()));
ArgumentCaptor<CreateTableRequest> captor = ArgumentCaptor.forClass(CreateTableRequest.class);
verify(glueClient, times(1)).createTable(captor.capture());
verify(wh).isDir(tblPath);
verify(wh, never()).mkdirs(tblPath, true);
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testCreateTableWithoutExistingDir() throws Exception {
Path tblPath = new Path(testTbl.getStorageDescriptor().getLocation());
setupMockWarehouseForPath(tblPath, false, true);
when(glueClient.getDatabase(new GetDatabaseRequest().withName(testDb.getName())))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTable(new GetTableRequest().withDatabaseName(testTbl.getDatabaseName())
.withName(testTbl.getName()))).thenThrow(new EntityNotFoundException(""));
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
metastoreClientDelegate.createTable(CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName()));
verify(glueClient, times(1)).createTable(any(CreateTableRequest.class));
verify(wh).isDir(tblPath);
verify(wh).mkdirs(tblPath, true);
}
@Test (expected = org.apache.hadoop.hive.metastore.api.AlreadyExistsException.class)
public void testCreateTableWithExistTable() throws Exception {
setupMockWarehouseForPath(new Path(testTbl.getStorageDescriptor().getLocation()), true, false);
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
metastoreClientDelegate.createTable(CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName()));
}
@Test
public void testAlterTable() throws Exception {
org.apache.hadoop.hive.metastore.api.Table newHiveTable
= CatalogToHiveConverter.convertTable(getTestTable(), testDb.getName());
newHiveTable.setTableName(testTbl.getName());
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(new GetDatabaseResult().withDatabase((testDb)));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable((testTbl)));
metastoreClientDelegateCatalogId.alterTable(testDb.getName(), testTbl.getName(), newHiveTable, null);
ArgumentCaptor<UpdateTableRequest> captor = ArgumentCaptor.forClass(UpdateTableRequest.class);
verify(glueClient, times(1)).updateTable(captor.capture());
TableInput expectedTableInput = GlueInputConverter.convertToTableInput(newHiveTable);
assertEquals(expectedTableInput, captor.getValue().getTableInput());
}
@Test(expected = UnsupportedOperationException.class)
public void testAlterTableRename() throws Exception {
org.apache.hadoop.hive.metastore.api.Table newHiveTable
= CatalogToHiveConverter.convertTable(getTestTable(), testDb.getName());
metastoreClientDelegate.alterTable(testDb.getName(), testTbl.getName(), newHiveTable, null);
}
@Test
public void testAlterTableSetExternalType() throws Exception {
org.apache.hadoop.hive.metastore.api.Table newHiveTable
= CatalogToHiveConverter.convertTable(getTestTable(), testDb.getName());
newHiveTable.setTableType(MANAGED_TABLE.toString());
newHiveTable.getParameters().put("EXTERNAL", "TRUE");
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(new GetDatabaseResult().withDatabase((testDb)));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable((testTbl)));
metastoreClientDelegate.alterTable(testDb.getName(), newHiveTable.getTableName(), newHiveTable, null);
ArgumentCaptor<UpdateTableRequest> captor = ArgumentCaptor.forClass(UpdateTableRequest.class);
verify(glueClient, times(1)).updateTable(captor.capture());
assertEquals(EXTERNAL_TABLE.toString(), captor.getValue().getTableInput().getTableType());
}
@Test
public void testAlterTableSetManagedType() throws Exception {
org.apache.hadoop.hive.metastore.api.Table newHiveTable
= CatalogToHiveConverter.convertTable(getTestTable(), testDb.getName());
newHiveTable.setTableType(EXTERNAL_TABLE.toString());
newHiveTable.getParameters().put("EXTERNAL", "FALSE");
when(glueClient.getDatabase(any(GetDatabaseRequest.class))).thenReturn(new GetDatabaseResult().withDatabase((testDb)));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable((testTbl)));
metastoreClientDelegate.alterTable(testDb.getName(), newHiveTable.getTableName(), newHiveTable, null);
ArgumentCaptor<UpdateTableRequest> captor = ArgumentCaptor.forClass(UpdateTableRequest.class);
verify(glueClient, times(1)).updateTable(captor.capture());
assertEquals(MANAGED_TABLE.toString(), captor.getValue().getTableInput().getTableType());
}
@Test(expected=UnsupportedOperationException.class)
public void testListTableNamesByFilter() throws Exception {
metastoreClientDelegate.listTableNamesByFilter("db","filter", (short)1);
}
@Test
public void testDropTableWithDeleteData() throws Exception {
Path tbl_path = new Path(testTbl.getStorageDescriptor().getLocation());
List<String> values = Lists.newArrayList("foo");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName()).withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
when(glueClient.getTable(new GetTableRequest()
.withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
.thenReturn(new GetTableResult().withTable(testTbl));
when(glueClient.deletePartition(new DeletePartitionRequest()
.withDatabaseName(testDb.getName()).withPartitionValues(values).withTableName(testTbl.getName())))
.thenReturn(new DeletePartitionResult());
when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
.thenReturn(new GetPartitionsResult().withPartitions(partition));
when(glueClient.getPartition(new GetPartitionRequest()
.withDatabaseName(testDb.getName()).withTableName(testTbl.getName()).withPartitionValues(values)))
.thenReturn(new GetPartitionResult().withPartition(partition));
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
metastoreClientDelegate.dropTable(testTbl.getDatabaseName(), testTbl.getName(), true, true, true);
verify(glueClient).deleteTable(new DeleteTableRequest().withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName()));
verify(wh).deleteDir(tbl_path, true, true);
}
@Test
public void testDropTableWithoutDeleteData() throws Exception {
Path tblPath = new Path(testTbl.getStorageDescriptor().getLocation());
List<String> values = Lists.newArrayList("foo");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName()).withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
when(glueClient.getTable(new GetTableRequest()
.withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
.thenReturn(new GetTableResult().withTable(testTbl));
when(glueClient.deletePartition(new DeletePartitionRequest()
.withDatabaseName(testDb.getName()).withPartitionValues(values).withTableName(testTbl.getName())))
.thenReturn(new DeletePartitionResult());
when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
.thenReturn(new GetPartitionsResult().withPartitions(partition));
when(glueClient.getPartition(new GetPartitionRequest()
.withDatabaseName(testDb.getName()).withTableName(testTbl.getName()).withPartitionValues(values)))
.thenReturn(new GetPartitionResult().withPartition(partition));
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
metastoreClientDelegate.dropTable(testTbl.getDatabaseName(), testTbl.getName(), false, true, true);
verify(glueClient).deleteTable(new DeleteTableRequest().withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName()));
verify(wh, never()).deleteDir(tblPath, true, true);
}
@Test
public void testDropExternalTableWithoutDeleteData() throws Exception {
Path tblPath = new Path(testTbl.getStorageDescriptor().getLocation());
List<String> values = Lists.newArrayList("foo");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName()).withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
testTbl.getParameters().put("EXTERNAL", "TRUE");
when(glueClient.getTable(new GetTableRequest()
.withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
.thenReturn(new GetTableResult().withTable(testTbl));
when(glueClient.deletePartition(new DeletePartitionRequest()
.withDatabaseName(testDb.getName()).withPartitionValues(values).withTableName(testTbl.getName())))
.thenReturn(new DeletePartitionResult());
when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
.thenReturn(new GetPartitionsResult().withPartitions(partition));
when(glueClient.getPartition(new GetPartitionRequest()
.withDatabaseName(testDb.getName()).withTableName(testTbl.getName()).withPartitionValues(values)))
.thenReturn(new GetPartitionResult().withPartition(partition));
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
metastoreClientDelegate.dropTable(testTbl.getDatabaseName(), testTbl.getName(), false, true, true);
verify(glueClient).deleteTable(new DeleteTableRequest().withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName()));
verify(wh, never()).deleteDir(tblPath, true, true);
}
@Test
public void testValidateTableAndCreateDirectoryVirtualView() throws Exception {
testTbl.setTableType(TableType.VIRTUAL_VIEW.toString());
testTbl.getStorageDescriptor().setLocation(null);
org.apache.hadoop.hive.metastore.api.Table hiveTbl = CatalogToHiveConverter.convertTable(testTbl, testTbl.getDatabaseName());
when(glueClient.getDatabase(any(GetDatabaseRequest.class)))
.thenReturn(new GetDatabaseResult().withDatabase(testDb));
when(glueClient.getTable(new GetTableRequest()
.withDatabaseName(testTbl.getDatabaseName()).withName(testTbl.getName())))
.thenThrow(EntityNotFoundException.class);
assertFalse(metastoreClientDelegate.validateNewTableAndCreateDirectory(hiveTbl));
assertNull(testTbl.getStorageDescriptor().getLocation());
verify(wh, never()).mkdirs(any(Path.class), anyBoolean());
}
// ======================= Partition =======================
@Test
public void testGetPartitionByValues() throws Exception {
List<String> values = Lists.newArrayList("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
GetPartitionRequest request = new GetPartitionRequest()
.withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withPartitionValues(values);
when(glueClient.getPartition(request)).thenReturn(new GetPartitionResult().withPartition(partition));
org.apache.hadoop.hive.metastore.api.Partition result = metastoreClientDelegate.getPartition(testDb.getName(), testTbl.getName(), values);
verify(glueClient, times(1)).getPartition(request);
assertThat(result.getValues(), is(values));
}
@Test
public void testGetPartitionByValuesWithCatalogId() throws Exception {
List<String> values = Lists.newArrayList("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
GetPartitionRequest request = new GetPartitionRequest()
.withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withPartitionValues(values).
withCatalogId(CATALOG_ID);
when(glueClient.getPartition(request)).thenReturn(new GetPartitionResult().withPartition(partition));
org.apache.hadoop.hive.metastore.api.Partition result = metastoreClientDelegateCatalogId.getPartition(testDb.getName(), testTbl.getName(), values);
ArgumentCaptor<GetPartitionRequest> captor = ArgumentCaptor.forClass(GetPartitionRequest.class);
verify(glueClient, times(1)).getPartition(captor.capture());
assertThat(result.getValues(), is(values));
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testGetPartitionByName() throws Exception {
String partitionName = "/a=foo/b=bar";
List<String> values = ImmutableList.of("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
when(glueClient.getPartition(any(GetPartitionRequest.class)))
.thenReturn(new GetPartitionResult().withPartition(partition));
org.apache.hadoop.hive.metastore.api.Partition result
= metastoreClientDelegate.getPartition(testDb.getName(), testTbl.getName(), partitionName);
verify(glueClient).getPartition(any(GetPartitionRequest.class));
assertThat(result.getValues(), is(values));
}
@Test(expected=NoSuchObjectException.class)
public void testGetPartitionEntityNotFound() throws Exception {
when(glueClient.getPartition(any(GetPartitionRequest.class)))
.thenThrow(new EntityNotFoundException("Test exception: partition not found"));
metastoreClientDelegate.getPartition(testDb.getName(), testTbl.getName(), "testPart");
verify(glueClient, times(1)).getPartition(any(GetPartitionRequest.class));
}
@Test
public void testGetPartitionsByNames() throws Exception {
String partitionName = "/a=foo/b=bar";
List<String> values = ImmutableList.of("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
when(glueClient.batchGetPartition(any(BatchGetPartitionRequest.class)))
.thenReturn(new BatchGetPartitionResult().withPartitions(partition));
List<org.apache.hadoop.hive.metastore.api.Partition> result
= metastoreClientDelegate.getPartitionsByNames(testDb.getName(), testTbl.getName(), ImmutableList.of(partitionName));
verify(glueClient, times(1)).batchGetPartition(any(BatchGetPartitionRequest.class));
assertNotNull(result);
assertThat(Iterables.getOnlyElement(result).getValues(), is(values));
}
@Test
public void testGetPartitionsByNamesWithCatalogId() throws Exception {
String partitionName = "/a=foo/b=bar";
List<String> values = ImmutableList.of("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
when(glueClient.batchGetPartition(any(BatchGetPartitionRequest.class)))
.thenReturn(new BatchGetPartitionResult().withPartitions(partition));
List<org.apache.hadoop.hive.metastore.api.Partition> result
= metastoreClientDelegateCatalogId.getPartitionsByNames(testDb.getName(), testTbl.getName(), ImmutableList.of(partitionName));
ArgumentCaptor<BatchGetPartitionRequest> captor = ArgumentCaptor.forClass(BatchGetPartitionRequest.class);
verify(glueClient, times(1)).batchGetPartition(captor.capture());
assertNotNull(result);
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testGetPartitionsByNamePropagateException() throws Exception {
String exceptionMessage = "Partition not found";
when(glueClient.batchGetPartition(any(BatchGetPartitionRequest.class)))
.thenThrow(new EntityNotFoundException(exceptionMessage));
try {
metastoreClientDelegate.getPartitionsByNames(testDb.getName(), testTbl.getName(), ImmutableList.of("/a=foo/b=bar"));
} catch (Exception e) {
assertThat(e, instanceOf(NoSuchObjectException.class));
assertThat(e.getMessage(), containsString(exceptionMessage));
}
verify(glueClient, times(1)).batchGetPartition(any(BatchGetPartitionRequest.class));
}
@Test
public void testGetPartitionsByNameTwoPages() throws Exception {
int numPartNames = BATCH_GET_PARTITIONS_MAX_REQUEST_SIZE + 10;
List<String> partNames = getTestPartitionNames(numPartNames);
when(glueClient.batchGetPartition(any(BatchGetPartitionRequest.class)))
.thenReturn(new BatchGetPartitionResult().withPartitions(ImmutableList.<Partition>of()));
metastoreClientDelegate.getPartitionsByNames(testDb.getName(), testTbl.getName(), partNames);
verify(glueClient, times(2)).batchGetPartition(any(BatchGetPartitionRequest.class));
}
private static List<String> getTestPartitionNames(int numPartitions) {
List<String> partNames = Lists.newArrayList();
for (int i = 1; i < numPartitions; i++) {
partNames.add(String.format("a=%d", i));
}
return partNames;
}
@Test
public void testGetPartitions() throws Exception {
List<String> expectedValues = Lists.newArrayList("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(expectedValues);
when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
.thenReturn(new GetPartitionsResult().withPartitions(Lists.newArrayList(partition)));
List<org.apache.hadoop.hive.metastore.api.Partition> res = metastoreClientDelegate.getPartitions(
testDb.getName(), testTbl.getName(), null, 10);
verify(glueClient, times(1)).getPartitions(any(GetPartitionsRequest.class));
assertThat(res, is(not(empty())));
List<String> values = Iterables.getOnlyElement(res).getValues();
assertThat(values, is(expectedValues));
}
@Test
public void testGetPartitionsParallel() throws Exception {
final int numSegments = 2;
HiveConf conf = new HiveConf(this.conf);
conf.setInt(GlueMetastoreClientDelegate.NUM_PARTITION_SEGMENTS_CONF, numSegments);
GlueMetastoreClientDelegate delegate = new GlueMetastoreClientDelegate(conf, new DefaultAWSGlueMetastore(conf, glueClient), wh);
final Set<List<String>> expectedValues = Sets.newHashSet();
final List<Partition> partitions = Lists.newArrayList();
final int numPartitions = DefaultAWSGlueMetastore.GET_PARTITIONS_MAX_SIZE + 10;
final int maxPartitionsToRequest = numPartitions - 1;
for (int i = 1; i <= numPartitions; i++) {
List<String> partitionKeys = Arrays.asList("keyA:" + i, "keyB:" + i);
if (i <= maxPartitionsToRequest) {
expectedValues.add(partitionKeys);
}
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(partitionKeys);
partitions.add(partition);
}
when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
.thenAnswer(new Answer<GetPartitionsResult>() {
@Override
public GetPartitionsResult answer(InvocationOnMock invocation) {
GetPartitionsRequest request = invocation.getArgumentAt(0, GetPartitionsRequest.class);
GetPartitionsResult result;
if (request.getSegment() == null) {
fail("Should pass in segment");
}
switch (request.getSegment().getSegmentNumber()) {
case 0:
result = new GetPartitionsResult().withPartitions(partitions.subList(0, numPartitions / 2));
break;
case 1:
result = new GetPartitionsResult().withPartitions(partitions.subList(numPartitions / 2, partitions.size()));
break;
default:
result = new GetPartitionsResult().withPartitions(Collections.<Partition>emptyList());
fail("Got segmentNumber >= " + numSegments);
}
return result;
}
});
List<org.apache.hadoop.hive.metastore.api.Partition> res = delegate.getPartitions(
testDb.getName(), testTbl.getName(), null, maxPartitionsToRequest);
verify(glueClient, times(numSegments))
.getPartitions(any(GetPartitionsRequest.class));
assertThat(res, is(not(empty())));
Iterable<List<String>> values = Iterables.transform(res,
new Function<org.apache.hadoop.hive.metastore.api.Partition, List<String>>() {
public List<String> apply(org.apache.hadoop.hive.metastore.api.Partition partition) {
return partition.getValues();
}
});
assertThat(Sets.newHashSet(values), is(expectedValues));
}
@Test(expected = MetaException.class)
public void testGetPartitionsPartialFailure() throws Exception {
List<String> partitionKeys1 = Arrays.asList("foo1", "bar1");
final Partition partition1 = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(partitionKeys1);
when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
.thenAnswer(new Answer<GetPartitionsResult>() {
@Override
public GetPartitionsResult answer(InvocationOnMock invocation) {
GetPartitionsRequest request = invocation.getArgumentAt(0, GetPartitionsRequest.class);
GetPartitionsResult result;
switch (request.getSegment().getSegmentNumber()) {
case 0:
result = new GetPartitionsResult().withPartitions(Lists.newArrayList(partition1));
break;
default:
throw new OperationTimeoutException("timeout");
}
return result;
}
});
List<org.apache.hadoop.hive.metastore.api.Partition> res = metastoreClientDelegate.getPartitions(
testDb.getName(), testTbl.getName(), null, -1);
}
@Test(expected = IllegalArgumentException.class)
public void testTooHighGluePartitionSegments() throws MetaException {
HiveConf conf = new HiveConf(this.conf);
conf.setInt(GlueMetastoreClientDelegate.NUM_PARTITION_SEGMENTS_CONF,
DefaultAWSGlueMetastore.MAX_NUM_PARTITION_SEGMENTS + 1);
GlueMetastoreClientDelegate delegate = new GlueMetastoreClientDelegate(conf, new DefaultAWSGlueMetastore(conf, glueClient), wh);
}
@Test
public void testDropPartitionUsingValues() throws Exception {
List<String> values = Lists.newArrayList("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
DeletePartitionRequest request = new DeletePartitionRequest()
.withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withPartitionValues(values);
when(glueClient.deletePartition(request)).thenReturn(new DeletePartitionResult());
when(glueClient.getPartition(any(GetPartitionRequest.class))).thenReturn(new GetPartitionResult().withPartition(partition));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
metastoreClientDelegate.dropPartition(testDb.getName(), testTbl.getName(), values, false, false, false);
verify(glueClient, times(1)).deletePartition(request);
}
@Test
public void testDropPartitionUsingValuesWithCatalogId() throws Exception {
List<String> values = Lists.newArrayList("foo", "bar");
Partition partition = new Partition().withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withValues(values)
.withStorageDescriptor(TestObjects.getTestStorageDescriptor());
DeletePartitionRequest request = new DeletePartitionRequest()
.withDatabaseName(testDb.getName())
.withTableName(testTbl.getName())
.withPartitionValues(values);
when(glueClient.deletePartition(request)).thenReturn(new DeletePartitionResult());
when(glueClient.getPartition(any(GetPartitionRequest.class))).thenReturn(new GetPartitionResult().withPartition(partition));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
metastoreClientDelegateCatalogId.dropPartition(testDb.getName(), testTbl.getName(), values, false, false, false);
ArgumentCaptor<DeletePartitionRequest> captor = ArgumentCaptor.forClass(DeletePartitionRequest.class);
verify(glueClient, times(1)).deletePartition(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testAppendPartition() throws Exception {
List<String> values = ImmutableList.of("foo");
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
Path partLocation = new Path(testTbl.getStorageDescriptor().getLocation(),
Warehouse.makePartName(CatalogToHiveConverter.convertFieldSchemaList(testTbl.getPartitionKeys()), values));
setupMockWarehouseForPath(partLocation, false, true);
mockBatchCreatePartitionsSucceed();
org.apache.hadoop.hive.metastore.api.Partition res =
metastoreClientDelegate.appendPartition(testDb.getName(), testTbl.getName(), values);
verify(wh, times(1)).mkdirs(partLocation, true);
assertThat(res.getValues(), is(values));
}
@Test
public void testAddPartitionsEmpty() throws Exception {
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = Lists.newArrayList();
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegate.addPartitions(partitions, false, true);
verify(glueClient, never()).getTable(any(GetTableRequest.class));
verify(glueClient, never()).batchCreatePartition(any(BatchCreatePartitionRequest.class));
assertThat(partitionsCreated, is(empty()));
assertDaemonThreadPools();
}
@Test
public void testAddPartitions() throws Exception {
mockBatchCreatePartitionsSucceed();
setupMockWarehouseForPath(new Path(testTbl.getStorageDescriptor().getLocation().toString()), false, true);
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegate.addPartitions(partitions, false, true);
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, never()).deleteDir(any(Path.class), eq(true));
assertEquals(numPartitions, partitionsCreated.size());
assertThat(partitionsCreated, containsInAnyOrder(partitions.toArray()));
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsEmptyPartitionLocation() throws Exception {
// Case: table contains location & partition location is empty.
// Test that created partitions contains location
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated = addPartitionsWithEmptyLocationsValid(numPartitions);
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
for (org.apache.hadoop.hive.metastore.api.Partition part : partitionsCreated) {
assertThat(part.getSd().getLocation(), notNullValue());
}
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsEmptyTableAndPartitionLocation() throws Exception {
// Case: table location is empty (VIRTUAL_VIEW) & partition location is empty.
// Test that created partitions does not contain location as these are Views.
testTbl.getStorageDescriptor().setLocation(null);
int numPartitions = 1;
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated = addPartitionsWithEmptyLocationsValid(numPartitions);
verify(wh, never()).mkdirs(any(Path.class), anyBoolean());
assertThat(partitionsCreated.get(0).getSd().getLocation(), nullValue());
assertDaemonThreadPools();
}
private List<org.apache.hadoop.hive.metastore.api.Partition> addPartitionsWithEmptyLocationsValid(int numPartitions) throws Exception {
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
for (org.apache.hadoop.hive.metastore.api.Partition partition : partitions) {
partition.getSd().setLocation(null);
}
mockBatchCreatePartitionsSucceed();
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
when(wh.mkdirs(any(Path.class), anyBoolean())).thenReturn(true);
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegate.addPartitions(partitions, false, true);
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, never()).deleteDir(any(Path.class), anyBoolean());
assertEquals(numPartitions, partitionsCreated.size());
assertThat(partitionsCreated, containsInAnyOrder(partitions.toArray()));
return partitionsCreated;
}
@Test(expected = MetaException.class)
public void testAddPartitions_PartitionViewWithLocation() throws Exception {
// Case: table location is empty (VIRTUAL_VIEW) with partition containing location
// In Hive, this throws MetaException because it doesn't allow parititon views to have location
Table table = testTbl;
table.getStorageDescriptor().setLocation(null);
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
mockBatchCreatePartitionsSucceed();
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(table));
when(wh.mkdirs(any(Path.class), anyBoolean())).thenReturn(true);
metastoreClientDelegate.addPartitions(partitions, false, true);
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsDoNotNeedResult() throws Exception {
mockBatchCreatePartitionsSucceed();
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegate.addPartitions(partitions, false, false);
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, never()).deleteDir(any(Path.class), eq(true));
assertThat(partitionsCreated, is(nullValue()));
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsTwoPages() throws Exception {
mockBatchCreatePartitionsSucceed();
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
int numPartitions = (int) (BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE * 1.2);
int expectedBatches = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegate.addPartitions(partitions, false, true);
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(expectedBatches)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, never()).deleteDir(any(Path.class), eq(true));
assertEquals(numPartitions, partitionsCreated.size());
assertThat(partitionsCreated, containsInAnyOrder(partitions.toArray()));
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsTwoPagesWithCatalogId() throws Exception {
mockBatchCreatePartitionsSucceed();
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
int numPartitions = (int) (BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE * 1.2);
int expectedBatches = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegateCatalogId.addPartitions(partitions, false, true);
ArgumentCaptor<BatchCreatePartitionRequest> captor = ArgumentCaptor.forClass(BatchCreatePartitionRequest.class);
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(expectedBatches)).batchCreatePartition(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, never()).deleteDir(any(Path.class), eq(true));
assertEquals(numPartitions, partitionsCreated.size());
assertThat(partitionsCreated, containsInAnyOrder(partitions.toArray()));
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsFailedServiceException() throws Exception {
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<String> values = partitions.get(0).getValues();
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenReturn(new BatchCreatePartitionResult().withErrors(TestObjects.getPartitionError(values,
new InternalServiceException("exception"))));
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
try {
metastoreClientDelegate.addPartitions(partitions, false, true);
fail("should throw");
} catch (Exception e) {
assertThat(e, is(instanceOf(MetaException.class)));
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, times(1)).deleteDir(any(Path.class), eq(true));
assertDaemonThreadPools();
}
}
@Test
public void testAddPartitionsFailedAlreadyExistsException() throws Exception {
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<String> values = ImmutableList.of("foo1");
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenReturn(new BatchCreatePartitionResult().withErrors(TestObjects.getPartitionError(values,
new com.amazonaws.services.glue.model.AlreadyExistsException("exception"))));
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
try {
metastoreClientDelegate.addPartitions(partitions, false, true);
fail("Should throw");
} catch (Exception e) {
assertThat(e, is(instanceOf(org.apache.hadoop.hive.metastore.api.AlreadyExistsException.class)));
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, times(1)).deleteDir(any(Path.class), eq(true));
assertDaemonThreadPools();
}
}
@Test
public void testAddPartitionsThrowsEntityNotFoundException() throws Exception {
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenThrow(new EntityNotFoundException("exception"));
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
try {
metastoreClientDelegate.addPartitions(partitions, false, true);
fail("Should throw");
} catch (Exception e) {
assertThat(e, is(instanceOf(NoSuchObjectException.class)));
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, times(numPartitions)).deleteDir(any(Path.class), eq(true));
assertDaemonThreadPools();
}
}
@Test
public void testAddPartitionsThrowsExceptionSecondPage() throws Exception {
int numPartitions = 200;
int secondPageSize = numPartitions - BATCH_CREATE_PARTITIONS_MAX_REQUEST_SIZE;
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenReturn(new BatchCreatePartitionResult())
.thenThrow(new InvalidInputException("exception"));
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
try {
metastoreClientDelegate.addPartitions(partitions, false, true);
fail("Should throw");
} catch (Exception e) {
assertThat(e, is(instanceOf(InvalidObjectException.class)));
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(2)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, times(secondPageSize)).deleteDir(any(Path.class), eq(true));
assertDaemonThreadPools();
}
}
@Test
public void testAddPartitionsIfNotExists() throws Exception {
List<String> values = ImmutableList.of("foo1");
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenReturn(new BatchCreatePartitionResult().withErrors(TestObjects.getPartitionError(values,
new com.amazonaws.services.glue.model.AlreadyExistsException("exception"))));
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
List<org.apache.hadoop.hive.metastore.api.Partition> partitionsCreated =
metastoreClientDelegate.addPartitions(partitions, true, true);
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, never()).deleteDir(any(Path.class), eq(true));
assertEquals(1, partitionsCreated.size());
assertThat(partitionsCreated.get(0), isIn(partitions));
assertDaemonThreadPools();
}
@Test
public void testAddPartitionsKeysAndValuesNotMatch() throws Exception {
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
//make the partition value size inconsistent with key size
partitions.get(1).setValues(Lists.newArrayList("foo1", "bar1"));
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
try {
metastoreClientDelegate.addPartitions(partitions, true, true);
fail("should throw");
} catch (IllegalArgumentException e) {
verify(wh, never()).getDnsPath(any(Path.class));
assertDaemonThreadPools();
}
}
@Test
public void testAddPartitionsDeleteAddedPathsWhenAddPathFail() throws Exception {
int numPartitions = 2;
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = getTestPartitions(numPartitions);
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
when(wh.isDir(any(Path.class))).thenReturn(false);
when(wh.mkdirs(any(Path.class), eq(true))).thenReturn(true).thenReturn(false); // succeed first, then fail
try {
metastoreClientDelegate.addPartitions(partitions, true, true);
fail("should throw");
} catch (MetaException e) {
verify(wh, times(numPartitions)).getDnsPath(any(Path.class));
verify(wh, times(numPartitions)).isDir(any(Path.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, times(1)).deleteDir(any(Path.class), eq(true));
assertDaemonThreadPools();
}
}
@Test
public void testAddPartitionsCallGetPartitionForInternalServiceException() throws Exception {
int numPartitions = 3;
String dbName = testDb.getName();
String tableName = testTbl.getName();
List<String> values1 = Lists.newArrayList("val1");
List<String> values2 = Lists.newArrayList("val2");
List<String> values3 = Lists.newArrayList("val3");
Partition partition1 = TestObjects.getTestPartition(dbName, tableName, values1);
Partition partition2 = TestObjects.getTestPartition(dbName, tableName, values2);
Partition partition3 = TestObjects.getTestPartition(dbName, tableName, values3);
List<Partition> partitions = Lists.newArrayList(partition1, partition2, partition3);
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenThrow(new InternalServiceException("InternalServiceException"));
when(glueClient.getTable(any(GetTableRequest.class)))
.thenReturn(new GetTableResult().withTable(testTbl));
when(glueClient.getPartition(new GetPartitionRequest()
.withDatabaseName(dbName)
.withTableName(tableName)
.withPartitionValues(partition1.getValues())))
.thenReturn(new GetPartitionResult().withPartition(partition1));
when(glueClient.getPartition(new GetPartitionRequest()
.withDatabaseName(dbName)
.withTableName(tableName)
.withPartitionValues(partition2.getValues())))
.thenThrow(new EntityNotFoundException("EntityNotFoundException"));
when(glueClient.getPartition(new GetPartitionRequest()
.withDatabaseName(dbName)
.withTableName(tableName)
.withPartitionValues(partition3.getValues())))
.thenThrow(new NullPointerException("NullPointerException"));
try {
metastoreClientDelegate.addPartitions(CatalogToHiveConverter.convertPartitions(partitions), false, true);
fail("Should throw");
} catch (Exception e) {
assertThat(e, is(instanceOf(MetaException.class)));
verify(glueClient, times(1)).getTable(any(GetTableRequest.class));
verify(glueClient, times(1)).batchCreatePartition(any(BatchCreatePartitionRequest.class));
verify(glueClient, times(numPartitions)).getPartition(any(GetPartitionRequest.class));
verify(wh, times(numPartitions)).mkdirs(any(Path.class), eq(true));
verify(wh, times(2)).deleteDir(any(Path.class), eq(true));
assertDaemonThreadPools();
}
}
private void mockBatchCreatePartitionsSucceed() {
when(glueClient.batchCreatePartition(any(BatchCreatePartitionRequest.class)))
.thenReturn(new BatchCreatePartitionResult());
}
private List<org.apache.hadoop.hive.metastore.api.Partition> getTestPartitions(int count) {
List<org.apache.hadoop.hive.metastore.api.Partition> partitions = Lists.newArrayList();
for (int i = 0; i < count; i++) {
List<String> values = ImmutableList.of("foo" + i);
Partition partition = TestObjects.getTestPartition(testDb.getName(), testTbl.getName(), values);
partitions.add(CatalogToHiveConverter.convertPartition(partition));
}
return partitions;
}
@Test
public void testAlterPartitions() throws Exception {
List<String> values = ImmutableList.of("foo", "bar");
Partition partition = getTestPartition(testTbl.getDatabaseName(), testTbl.getName(), values);
org.apache.hadoop.hive.metastore.api.Partition hivePartition = CatalogToHiveConverter.convertPartition(partition);
PartitionInput input = GlueInputConverter.convertToPartitionInput(partition);
UpdatePartitionRequest request = new UpdatePartitionRequest()
.withDatabaseName(testTbl.getDatabaseName())
.withTableName(testTbl.getName())
.withPartitionInput(input)
.withPartitionValueList(partition.getValues());
when(glueClient.updatePartition(request)).thenReturn(new UpdatePartitionResult());
metastoreClientDelegate.alterPartitions(testDb.getName(), testTbl.getName(), ImmutableList.of(hivePartition));
verify(glueClient, times(1)).updatePartition(any(UpdatePartitionRequest.class));
}
@Test
public void testAlterParititonDDLTimeUpdated() throws Exception {
List<String> values = ImmutableList.of("foo", "bar");
org.apache.hadoop.hive.metastore.api.Partition partition
= CatalogToHiveConverter.convertPartition(getTestPartition(testTbl.getDatabaseName(), testTbl.getName(), values));
metastoreClientDelegate.alterPartitions(testTbl.getDatabaseName(), testTbl.getName(), Lists.newArrayList(partition));
ArgumentCaptor<UpdatePartitionRequest> captor = ArgumentCaptor.forClass(UpdatePartitionRequest.class);
verify(glueClient, times(1)).updatePartition(captor.capture());
assertTrue(captor.getValue().getPartitionInput().getParameters().containsKey(hive_metastoreConstants.DDL_TIME));
}
// =================== Roles & Privilege ===================
@Test(expected=UnsupportedOperationException.class)
public void testGrantPublicRole() throws Exception {
metastoreClientDelegate.grantRole("public", "user",
org.apache.hadoop.hive.metastore.api.PrincipalType.USER, "grantor",
org.apache.hadoop.hive.metastore.api.PrincipalType.ROLE, true);
}
@Test(expected=UnsupportedOperationException.class)
public void testRevokeRole() throws Exception {
metastoreClientDelegate.revokeRole("role", "user",
org.apache.hadoop.hive.metastore.api.PrincipalType.USER, true);
}
@Test(expected=UnsupportedOperationException.class)
public void testCreateRole() throws Exception {
metastoreClientDelegate.createRole(new org.apache.hadoop.hive.metastore.api.Role(
"role", (int) (new Date().getTime() / 1000), "owner"));
}
@Test(expected=UnsupportedOperationException.class)
public void testCreatePublicRole() throws Exception {
metastoreClientDelegate.createRole(new org.apache.hadoop.hive.metastore.api.Role(
"public", (int) (new Date().getTime() / 1000), "owner"));
}
@Test(expected=UnsupportedOperationException.class)
public void testDropRole() throws Exception {
metastoreClientDelegate.dropRole("role");
}
@Test(expected=UnsupportedOperationException.class)
public void testDropPublicRole() throws Exception {
metastoreClientDelegate.dropRole("public");
}
@Test(expected=UnsupportedOperationException.class)
public void testDropAdminRole() throws Exception {
metastoreClientDelegate.dropRole("admin");
}
@Test(expected=UnsupportedOperationException.class)
public void testListRolesWithRolePrincipalType() throws Exception {
metastoreClientDelegate.listRoles("user", PrincipalType.ROLE);
}
@Test(expected=UnsupportedOperationException.class)
public void testGetPrincipalsInRole() throws Exception {
metastoreClientDelegate.getPrincipalsInRole(
new org.apache.hadoop.hive.metastore.api.GetPrincipalsInRoleRequest("role"));
}
@Test(expected=UnsupportedOperationException.class)
public void testRoleGrantsForPrincipal() throws Exception {
metastoreClientDelegate.getRoleGrantsForPrincipal(
new org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest("user",
org.apache.hadoop.hive.metastore.api.PrincipalType.USER));
}
@Test(expected=UnsupportedOperationException.class)
public void testGrantRole() throws Exception {
metastoreClientDelegate.grantRole("role", "user",
org.apache.hadoop.hive.metastore.api.PrincipalType.USER, "grantor",
org.apache.hadoop.hive.metastore.api.PrincipalType.ROLE, true);
}
@Test(expected=UnsupportedOperationException.class)
public void testGrantPrivileges() throws Exception {
metastoreClientDelegate.grantPrivileges(TestObjects.getPrivilegeBag());
}
@Test(expected=UnsupportedOperationException.class)
public void testRevokePrivileges() throws Exception {
metastoreClientDelegate.revokePrivileges(TestObjects.getPrivilegeBag(), false);
}
@Test(expected=UnsupportedOperationException.class)
public void testListPrivileges() throws Exception {
String principal = "user1";
org.apache.hadoop.hive.metastore.api.PrincipalType principalType =
org.apache.hadoop.hive.metastore.api.PrincipalType.USER;
metastoreClientDelegate.listPrivileges(principal, principalType, TestObjects.getHiveObjectRef());
}
@Test
public void testGetPrincipalPrivilegeSet() throws Exception {
String user = "user1";
List<String> groupList = ImmutableList.of();
org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet privilegeSet = metastoreClientDelegate
.getPrivilegeSet(TestObjects.getHiveObjectRef(), user, groupList);
assertThat(privilegeSet, is(nullValue()));
}
@Test(expected=UnsupportedOperationException.class)
public void testGrantPrivilegesThrowingMetaException() throws Exception {
metastoreClientDelegate.grantPrivileges(TestObjects.getPrivilegeBag());
}
// ====================== Statistics ======================
@Test(expected=UnsupportedOperationException.class)
public void testDeletePartitionColumnStatisticsValid() throws Exception {
String databaseName = "database-name";
String tableName = "table-name";
String partitionName = "A=a/B=b";
String columnName = "column-name";
metastoreClientDelegate.deletePartitionColumnStatistics(databaseName, tableName, partitionName, columnName);
}
@Test(expected=UnsupportedOperationException.class)
public void testDeleteTableColumnStatistics() throws Exception {
String databaseName = "database-name";
String tableName = "table-name";
String columnName = "column-name";
metastoreClientDelegate.deleteTableColumnStatistics(databaseName, tableName, columnName);
}
@Test(expected=UnsupportedOperationException.class)
public void testGetPartitionColumnStatisticsValid() throws Exception {
String databaseName = "database-name";
String tableName = "table-name";
List<String> partitionNames = ImmutableList.of("A=a/B=b", "A=x/B=y");
List<String> columnNames = ImmutableList.of("decimal-column", "string-column");
metastoreClientDelegate.getPartitionColumnStatistics(databaseName, tableName, partitionNames, columnNames);
}
@Test(expected=UnsupportedOperationException.class)
public void testGetTableColumnStatistics() throws Exception {
String databaseName = "database-name";
String tableName = "table-name";
List<String> columnNames = ImmutableList.of("decimal-column", "string-column");
metastoreClientDelegate.getTableColumnStatistics(databaseName, tableName, columnNames);
}
@Test(expected=UnsupportedOperationException.class)
public void testUpdatePartitionColumnStatistics() throws Exception {
org.apache.hadoop.hive.metastore.api.ColumnStatistics columnStatistics =
TestObjects.getHivePartitionColumnStatistics();
metastoreClientDelegate.updatePartitionColumnStatistics(columnStatistics);
}
@Test(expected=UnsupportedOperationException.class)
public void testUpdateTableColumnStatistics() throws Exception {
org.apache.hadoop.hive.metastore.api.ColumnStatistics columnStatistics =
TestObjects.getHiveTableColumnStatistics();
metastoreClientDelegate.updateTableColumnStatistics(columnStatistics);
}
private void assertDaemonThreadPools() {
String threadNameCreatePrefix =
GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.substring(
0,
GlueMetastoreClientDelegate.GLUE_METASTORE_DELEGATE_THREADPOOL_NAME_FORMAT.indexOf('%'));
for (Thread thread : Thread.getAllStackTraces().keySet()) {
String threadName = thread.getName();
if (threadName != null && threadName.startsWith(threadNameCreatePrefix)) {
assertTrue(thread.isDaemon());
}
}
}
//==================== Functions =====================
@Test
public void getFunction() throws Exception {
UserDefinedFunction udf = createUserDefinedFunction();
when(glueClient.getUserDefinedFunction(any(GetUserDefinedFunctionRequest.class))).thenReturn(
new GetUserDefinedFunctionResult().withUserDefinedFunction(udf));
metastoreClientDelegateCatalogId.getFunction(testDb.getName(), "test-func");
ArgumentCaptor<GetUserDefinedFunctionRequest> captor = ArgumentCaptor.forClass(GetUserDefinedFunctionRequest.class);
verify(glueClient, times(1)).getUserDefinedFunction(captor.capture());
GetUserDefinedFunctionRequest request = captor.getValue();
assertEquals(CATALOG_ID, request.getCatalogId());
assertEquals(testDb.getName(), request.getDatabaseName());
assertEquals("test-func", request.getFunctionName());
}
@Test
public void getFunctions() throws Exception {
UserDefinedFunction udf1 = createUserDefinedFunction();
UserDefinedFunction udf2 = createUserDefinedFunction();
List<UserDefinedFunction> udfList = new ArrayList<>();
udfList.add(udf1);
udfList.add(udf2);
when(glueClient.getUserDefinedFunctions(any(GetUserDefinedFunctionsRequest.class))).thenReturn(
new GetUserDefinedFunctionsResult().withUserDefinedFunctions(udfList).withNextToken(null));
List<String> result = metastoreClientDelegateCatalogId.getFunctions(testDb.getName(), "test-func");
ArgumentCaptor<GetUserDefinedFunctionsRequest> captor = ArgumentCaptor
.forClass(GetUserDefinedFunctionsRequest.class);
verify(glueClient, times(1)).getUserDefinedFunctions(captor.capture());
GetUserDefinedFunctionsRequest request = captor.getValue();
assertEquals(CATALOG_ID, request.getCatalogId());
assertEquals(testDb.getName(), request.getDatabaseName());
assertEquals("test-func", request.getPattern());
assertEquals(2, result.size());
}
@Test
public void testCreateFunction() throws Exception {
org.apache.hadoop.hive.metastore.api.Function hiveFunction = createHiveFunction();
metastoreClientDelegateCatalogId.createFunction(hiveFunction);
ArgumentCaptor<CreateUserDefinedFunctionRequest> captor = ArgumentCaptor
.forClass(CreateUserDefinedFunctionRequest.class);
verify(glueClient, times(1)).createUserDefinedFunction(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testDropFunction() throws Exception {
metastoreClientDelegateCatalogId.dropFunction(testDb.getName(), "test-func");
ArgumentCaptor<DeleteUserDefinedFunctionRequest> captor = ArgumentCaptor
.forClass(DeleteUserDefinedFunctionRequest.class);
verify(glueClient, times(1)).deleteUserDefinedFunction(captor.capture());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
@Test
public void testAlterFunction() throws Exception {
org.apache.hadoop.hive.metastore.api.Function hiveFunction = createHiveFunction();
metastoreClientDelegateCatalogId.alterFunction(testDb.getName(), "test-func", createHiveFunction());
ArgumentCaptor<UpdateUserDefinedFunctionRequest> captor = ArgumentCaptor
.forClass(UpdateUserDefinedFunctionRequest.class);
verify(glueClient, times(1)).updateUserDefinedFunction(captor.capture());
UpdateUserDefinedFunctionRequest request = captor.getValue();
assertEquals(testDb.getName(), request.getDatabaseName());
assertEquals("test-func", request.getFunctionName());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
}
private org.apache.hadoop.hive.metastore.api.Function createHiveFunction() {
org.apache.hadoop.hive.metastore.api.Function hiveFunction = new org.apache.hadoop.hive.metastore.api.Function();
hiveFunction.setClassName("testClass");
hiveFunction.setFunctionName("test-func");
hiveFunction.setOwnerName("test-owner");
hiveFunction.setOwnerType(org.apache.hadoop.hive.metastore.api.PrincipalType.USER);
return hiveFunction;
}
private UserDefinedFunction createUserDefinedFunction() {
UserDefinedFunction udf = new UserDefinedFunction();
udf.setFunctionName("test-func");
udf.setClassName("test-class");
udf.setCreateTime(new Date());
udf.setOwnerName("test-owner");
udf.setOwnerType(com.amazonaws.services.glue.model.PrincipalType.USER.name());
return udf;
}
// ==================== Schema =====================
@Test
public void testGetFields() throws Exception {
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
List<FieldSchema> res = metastoreClientDelegateCatalogId.getFields(testDb.getName(), testTbl.getName());
ArgumentCaptor<GetTableRequest> captor = ArgumentCaptor.forClass(GetTableRequest.class);
verify(glueClient, times(1)).getTable(captor.capture());
GetTableRequest request = captor.getValue();
assertEquals(testDb.getName(), request.getDatabaseName());
assertEquals(testTbl.getName(), request.getName());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
assertEquals(1, res.size());
}
@Test
public void testGetSchema() throws Exception {
when(glueClient.getTable(any(GetTableRequest.class))).thenReturn(new GetTableResult().withTable(testTbl));
List<FieldSchema> res = metastoreClientDelegateCatalogId.getFields(testDb.getName(), testTbl.getName());
ArgumentCaptor<GetTableRequest> captor = ArgumentCaptor.forClass(GetTableRequest.class);
verify(glueClient, times(1)).getTable(captor.capture());
GetTableRequest request = captor.getValue();
assertEquals(testDb.getName(), request.getDatabaseName());
assertEquals(testTbl.getName(), request.getName());
assertEquals(CATALOG_ID, captor.getValue().getCatalogId());
assertEquals(1, res.size());
}
}
