com.amazonaws.services.glue.model.Database Java Examples

@Test
public void doListSchemaNamesGlue()
        throws Exception
{
    GetDatabasesResult result = new GetDatabasesResult().withDatabaseList(
            new Database().withName(DEFAULT_SCHEMA),
            new Database().withName("ddb").withLocationUri(DYNAMO_DB_FLAG),
            new Database().withName("s3").withLocationUri("blah"));

    when(glueClient.getDatabases(any())).thenReturn(result);

    ListSchemasRequest req = new ListSchemasRequest(TEST_IDENTITY, TEST_QUERY_ID, TEST_CATALOG_NAME);
    ListSchemasResponse res = handler.doListSchemaNames(allocator, req);

    logger.info("doListSchemas - {}", res.getSchemas());

    assertThat(res.getSchemas().size(), equalTo(2));
    assertThat(res.getSchemas().contains("default"), is(true));
    assertThat(res.getSchemas().contains("ddb"), is(true));
}
 

/**
 * @return boolean
 *    true  -> directory created
 *    false -> directory not created
 */
public boolean validateNewTableAndCreateDirectory(org.apache.hadoop.hive.metastore.api.Table tbl) throws TException {
  checkNotNull(tbl, "tbl cannot be null");
  if (tableExists(tbl.getDbName(), tbl.getTableName())) {
    throw new AlreadyExistsException("Table " + tbl.getTableName() + " already exists.");
  }
  validateTableObject(tbl, conf);

  if (TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
    // we don't need to create directory for virtual views
    return false;
  }

  if (StringUtils.isEmpty(tbl.getSd().getLocation())) {
    org.apache.hadoop.hive.metastore.api.Database db = getDatabase(tbl.getDbName());
    tbl.getSd().setLocation(hiveShims.getDefaultTablePath(db, tbl.getTableName(), wh).toString());
  } else {
    tbl.getSd().setLocation(wh.getDnsPath(new Path(tbl.getSd().getLocation())).toString());
  }

  Path tblPath = new Path(tbl.getSd().getLocation());
  return makeDirs(wh, tblPath);
}
 

/**
 * Returns a list of schemas (aka databases) from AWS Glue DataCatalog with optional filtering.
 *
 * @param blockAllocator Tool for creating and managing Apache Arrow Blocks.
 * @param request Provides details on who made the request and which Athena catalog they are querying.
 * @param filter The DatabaseFilter to apply to all schemas (aka databases) before adding them to the results list.
 * @return The ListSchemasResponse which mostly contains the list of schemas (aka databases).
 */
protected ListSchemasResponse doListSchemaNames(BlockAllocator blockAllocator, ListSchemasRequest request, DatabaseFilter filter)
        throws Exception
{
    GetDatabasesRequest getDatabasesRequest = new GetDatabasesRequest();
    getDatabasesRequest.setCatalogId(getCatalog(request));

    List<String> schemas = new ArrayList<>();
    String nextToken = null;
    do {
        getDatabasesRequest.setNextToken(nextToken);
        GetDatabasesResult result = awsGlue.getDatabases(getDatabasesRequest);

        for (Database next : result.getDatabaseList()) {
            if (filter == null || filter.filter(next)) {
                schemas.add(next.getName());
            }
        }

        nextToken = result.getNextToken();
    }
    while (nextToken != null);

    return new ListSchemasResponse(request.getCatalogName(), schemas);
}
 

@Test
public void testGetDatabaseWhenCacheEnabledAndCacheMiss() {
    Database db = new Database();
    AWSGlueMetastoreCacheDecorator cacheDecorator =
            new AWSGlueMetastoreCacheDecorator(hiveConf, glueMetastore);
    assertNotNull(cacheDecorator.databaseCache);
    Cache dbCache = mock(Cache.class);
    cacheDecorator.databaseCache = dbCache;

    when(dbCache.getIfPresent(DB_NAME)).thenReturn(null);
    when(glueMetastore.getDatabase(DB_NAME)).thenReturn(db);
    doNothing().when(dbCache).put(DB_NAME, db);

    assertEquals(db, cacheDecorator.getDatabase(DB_NAME));

    verify(glueMetastore, times(1)).getDatabase(DB_NAME);
    verify(dbCache, times(1)).getIfPresent(DB_NAME);
    verify(dbCache, times(1)).put(DB_NAME, db);
}
 

@Override
public Database getDatabase(String dbName) {
    Database result;
    if(databaseCacheEnabled) {
        Database valueFromCache = databaseCache.getIfPresent(dbName);
        if(valueFromCache != null) {
            logger.info("Cache hit for operation [getDatabase] on key [" + dbName + "]");
            result = valueFromCache;
        } else {
            logger.info("Cache miss for operation [getDatabase] on key [" + dbName + "]");
            result = super.getDatabase(dbName);
            databaseCache.put(dbName, result);
        }
    } else {
        result = super.getDatabase(dbName);
    }
    return result;
}
 

@Test
public void testGetDatabaseWhenCacheDisabled() {
    //disable cache
    when(hiveConf.getBoolean(AWS_GLUE_DB_CACHE_ENABLE, false)).thenReturn(false);
    Database db = new Database();
    AWSGlueMetastoreCacheDecorator cacheDecorator =
            new AWSGlueMetastoreCacheDecorator(hiveConf, glueMetastore);
    when(glueMetastore.getDatabase(DB_NAME)).thenReturn(db);
    assertEquals(db, cacheDecorator.getDatabase(DB_NAME));
    assertNull(cacheDecorator.databaseCache);
    verify(glueMetastore, times(1)).getDatabase(DB_NAME);
}
 

@Test
public void testConvertDatabase()
{
    io.prestosql.plugin.hive.metastore.Database prestoDatabase = GlueToPrestoConverter.convertDatabase(testDatabase);
    assertEquals(prestoDatabase.getDatabaseName(), testDatabase.getName());
    assertEquals(prestoDatabase.getLocation().get(), testDatabase.getLocationUri());
    assertEquals(prestoDatabase.getComment().get(), testDatabase.getDescription());
    assertEquals(prestoDatabase.getParameters(), testDatabase.getParameters());
    assertEquals(prestoDatabase.getOwnerName(), PUBLIC_OWNER);
    assertEquals(prestoDatabase.getOwnerType(), PrincipalType.ROLE);
}
 

public static io.prestosql.plugin.hive.metastore.Database getPrestoTestDatabase()
{
    return io.prestosql.plugin.hive.metastore.Database.builder()
            .setDatabaseName("test-db" + generateRandom())
            .setComment(Optional.of("database desc"))
            .setLocation(Optional.of("/db"))
            .setParameters(ImmutableMap.of())
            .setOwnerName("PUBLIC")
            .setOwnerType(PrincipalType.ROLE).build();
}
 

public static Database getGlueTestDatabase()
{
    return new Database()
            .withName("test-db" + generateRandom())
            .withDescription("database desc")
            .withLocationUri("/db")
            .withParameters(ImmutableMap.of());
}
 

@Test
public void doListSchemaNames()
        throws Exception
{
    List<Database> databases = new ArrayList<>();
    databases.add(new Database().withName("db1"));
    databases.add(new Database().withName("db2"));

    when(mockGlue.getDatabases(any(GetDatabasesRequest.class)))
            .thenAnswer((InvocationOnMock invocationOnMock) ->
            {
                GetDatabasesRequest request = (GetDatabasesRequest) invocationOnMock.getArguments()[0];
                assertEquals(accountId, request.getCatalogId());
                GetDatabasesResult mockResult = mock(GetDatabasesResult.class);
                if (request.getNextToken() == null) {
                    when(mockResult.getDatabaseList()).thenReturn(databases);
                    when(mockResult.getNextToken()).thenReturn("next");
                }
                else {
                    //only return real info on 1st call
                    when(mockResult.getDatabaseList()).thenReturn(new ArrayList<>());
                    when(mockResult.getNextToken()).thenReturn(null);
                }
                return mockResult;
            });

    ListSchemasRequest req = new ListSchemasRequest(IdentityUtil.fakeIdentity(), queryId, catalog);
    ListSchemasResponse res = handler.doListSchemaNames(allocator, req);

    logger.info("doListSchemas - {}", res.getSchemas());

    assertEquals(databases.stream().map(next -> next.getName()).collect(Collectors.toList()),
            new ArrayList<>(res.getSchemas()));

    verify(mockGlue, times(2)).getDatabases(any(GetDatabasesRequest.class));
}
 

@Test
public void testDeleteDatabaseWhenCacheEnabled() {
    DatabaseInput dbInput = new DatabaseInput();
    AWSGlueMetastoreCacheDecorator cacheDecorator =
            new AWSGlueMetastoreCacheDecorator(hiveConf, glueMetastore);
    cacheDecorator.databaseCache.put(DB_NAME, new Database());
    doNothing().when(glueMetastore).deleteDatabase(DB_NAME);

    cacheDecorator.deleteDatabase(DB_NAME);

    //db should have been removed from cache
    assertNull(cacheDecorator.databaseCache.getIfPresent(DB_NAME));
    verify(glueMetastore, times(1)).deleteDatabase(DB_NAME);
}
 

@Test
public void testUpdateDatabaseWhenCacheEnabled() {
    DatabaseInput dbInput = new DatabaseInput();
    AWSGlueMetastoreCacheDecorator cacheDecorator =
            new AWSGlueMetastoreCacheDecorator(hiveConf, glueMetastore);
    cacheDecorator.databaseCache.put(DB_NAME, new Database());
    doNothing().when(glueMetastore).updateDatabase(DB_NAME, dbInput);

    cacheDecorator.updateDatabase(DB_NAME, dbInput);

    //db should have been removed from cache
    assertNull(cacheDecorator.databaseCache.getIfPresent(DB_NAME));
    verify(glueMetastore, times(1)).updateDatabase(DB_NAME, dbInput);
}
 

@Test
public void testGetDatabaseWhenCacheEnabledAndCacheHit() {
    Database db = new Database();
    AWSGlueMetastoreCacheDecorator cacheDecorator =
            new AWSGlueMetastoreCacheDecorator(hiveConf, glueMetastore);
    assertNotNull(cacheDecorator.databaseCache);
    Cache dbCache = mock(Cache.class);
    cacheDecorator.databaseCache = dbCache;

    when(dbCache.getIfPresent(DB_NAME)).thenReturn(db);

    assertEquals(db, cacheDecorator.getDatabase(DB_NAME));

    verify(dbCache, times(1)).getIfPresent(DB_NAME);
}
 

/**
 *
 * @return a test db
 */
public static Database getTestDatabase() {

  Map<String, String> parameters = Maps.newHashMap();
  parameters.put("param1", "value1");
  parameters.put("param2", "value2");

  Database database = new Database()
  	.withName("test-db-" + UUID.randomUUID().toString().replaceAll("[^a-zA-Z0-9]+", ""))
      .withDescription("database desc")
      .withLocationUri("/db")
      .withParameters(parameters);

  return database;
}
 

@Test
public void testConvertHiveDbToDatabaseInput() {
  org.apache.hadoop.hive.metastore.api.Database hivedb = CatalogToHiveConverter.convertDatabase(testDB);
  DatabaseInput dbInput = GlueInputConverter.convertToDatabaseInput(hivedb);

  assertEquals(testDB.getName(), dbInput.getName());
  assertEquals(testDB.getDescription(), dbInput.getDescription());
  assertEquals(testDB.getLocationUri(), dbInput.getLocationUri());
  assertEquals(testDB.getParameters(), dbInput.getParameters());
}
 

@Test
public void testDatabaseConversionWithNullFields() {
  Database catalogDb = TestObjects.getTestDatabase();
  catalogDb.setLocationUri(null);
  catalogDb.setParameters(null);
  org.apache.hadoop.hive.metastore.api.Database hiveDatabase = CatalogToHiveConverter
      .convertDatabase(catalogDb);
  assertThat(hiveDatabase.getLocationUri(), is(""));
  assertNotNull(hiveDatabase.getParameters());
}
 

@Test
public void testDatabaseConversion() {
  Database catalogDb = TestObjects.getTestDatabase();
  org.apache.hadoop.hive.metastore.api.Database hiveDatabase = CatalogToHiveConverter
      .convertDatabase(catalogDb);
  Database catalogDb2 = HiveToCatalogConverter.convertDatabase(hiveDatabase);
  assertEquals(catalogDb, catalogDb2);
}
 

@Override
public List<Database> getAllDatabases() {
    List<Database> ret = Lists.newArrayList();
    String nextToken = null;
    do {
        GetDatabasesRequest getDatabasesRequest = new GetDatabasesRequest().withNextToken(nextToken).withCatalogId(
                catalogId);
        GetDatabasesResult result = glueClient.getDatabases(getDatabasesRequest);
        nextToken = result.getNextToken();
        ret.addAll(result.getDatabaseList());
    } while (nextToken != null);
    return ret;
}
 

@Override
public Database getDatabase(String dbName) {
    GetDatabaseRequest getDatabaseRequest = new GetDatabaseRequest().withCatalogId(catalogId).withName(dbName);
    GetDatabaseResult result = glueClient.getDatabase(getDatabaseRequest);
    return result.getDatabase();
}
 

@Override
public List<Database> getAllDatabases() {
    return awsGlueMetastore.getAllDatabases();
}
 

@Override
public Database getDatabase(String dbName) {
    return awsGlueMetastore.getDatabase(dbName);
}
 

/**
 * Used to filter database results.
 *
 * @param database The database to evaluate.
 * @return True if the provided database should be in the result, False if not.
 */
boolean filter(Database database);
 

List<Database> getAllDatabases(); 

Database getDatabase(String dbName);