org.apache.hadoop.hive.metastore.api.TableMeta Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.TableMeta. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StaticDatabaseMappingServiceTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void panopticOperationsHandlerGetTableMeta() throws Exception {
  String pattern = "pattern";
  TableMeta primaryTableMeta = new TableMeta(PRIMARY_DB, "tbl", null);
  TableMeta federatedTableMeta = new TableMeta(FEDERATED_DB, "tbl", null);
  TableMeta ignoredTableMeta = new TableMeta("non_mapped_db", "tbl", null);

  when(primaryDatabaseClient.get_table_meta(pattern, pattern, null))
      .thenReturn(Collections.singletonList(primaryTableMeta));
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta(pattern, pattern, null))
      .thenReturn(Arrays.asList(federatedTableMeta, ignoredTableMeta));

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> expected = Lists.newArrayList(primaryTableMeta, federatedTableMeta);
  List<TableMeta> result = handler.getTableMeta(pattern, pattern, null);
  assertThat(result, is(expected));
}
 
Example #2
Source File: GlueMetastoreClientDelegate.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
public List<TableMeta> getTableMeta(
    String dbPatterns,
    String tablePatterns,
    List<String> tableTypes
) throws TException  {
  List<TableMeta> tables = new ArrayList<>();
  List<String> databases = getDatabases(dbPatterns);
  for (String dbName : databases) {
    String nextToken = null;
    List<Table> dbTables = glueMetastore.getTables(dbName, tablePatterns);
    for (Table catalogTable : dbTables) {
      if (tableTypes == null ||
          tableTypes.isEmpty() ||
          tableTypes.contains(catalogTable.getTableType())) {
        tables.add(CatalogToHiveConverter.convertTableMeta(catalogTable, dbName));
      }
    }
  }
  return tables;
}
 
Example #3
Source File: PrefixBasedDatabaseMappingServiceTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void panopticOperationsHandlerGetTableMetaWithNonWhitelistedDb() throws TException {
  List<String> tblTypes = Lists.newArrayList();
  TableMeta tableMeta = new TableMeta("federated_db", "tbl", null);
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta("federated_*", "*", tblTypes))
      .thenReturn(Lists.newArrayList(tableMeta));

  // set metastore whitelist to be nonempty
  federatedMetastore.setMappedDatabases(Collections.singletonList("testName"));
  service = new PrefixBasedDatabaseMappingService(metaStoreMappingFactory,
      Arrays.asList(primaryMetastore, federatedMetastore), queryMapping);

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> tableMetas = handler.getTableMeta("name_federated_*", "*", tblTypes);

  assertThat(tableMetas.size(), is(0));
}
 
Example #4
Source File: PrefixBasedDatabaseMappingServiceTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void panopticOperationsHandlerGetTableMeta() throws Exception {
  TableMeta federatedTableMeta = new TableMeta("federated_db", "tbl", null);
  TableMeta primaryTableMeta = new TableMeta("primary_db", "tbl", null);

  when(primaryDatabaseClient.get_table_meta("*_db", "*", null))
      .thenReturn(Collections.singletonList(primaryTableMeta));
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta("*_db", "*", null))
      .thenReturn(Collections.singletonList(federatedTableMeta));
  when(metaStoreMappingFederated.transformOutboundDatabaseName("federated_db")).thenReturn("name_federated_db");

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> expected = Arrays.asList(primaryTableMeta, federatedTableMeta);
  List<TableMeta> result = handler.getTableMeta("*_db", "*", null);
  assertThat(result, is(expected));
}
 
Example #5
Source File: GlueMetastoreClientDelegateTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
@Test
public void testGetTableMetaNullEmptyTableType() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = null;

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
    new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
    new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));

  tableTypes = Lists.newArrayList();
  tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 
Example #6
Source File: PanopticOperationHandler.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
protected List<TableMeta> getTableMeta(
    String tablePatterns,
    List<String> tableTypes,
    Map<DatabaseMapping, String> databaseMappingsForPattern,
    BiFunction<TableMeta, DatabaseMapping, Boolean> filter) {
  List<GetTableMetaRequest> allRequests = new ArrayList<>();

  for (Entry<DatabaseMapping, String> mappingWithPattern : databaseMappingsForPattern.entrySet()) {
    DatabaseMapping mapping = mappingWithPattern.getKey();
    GetTableMetaRequest tableMetaRequest = new GetTableMetaRequest(mapping, mappingWithPattern.getValue(),
        tablePatterns, tableTypes, filter);
    allRequests.add(tableMetaRequest);
  }

  List<TableMeta> result = getPanopticOperationExecutor()
      .executeRequests(allRequests, GET_TABLE_META_TIMEOUT, "Got exception fetching get_table_meta: {}");
  return result;
}
 
Example #7
Source File: CatalogToHiveConverter.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
public static TableMeta convertTableMeta(com.amazonaws.services.glue.model.Table catalogTable, String dbName) {
  TableMeta tableMeta = new TableMeta();
  tableMeta.setDbName(dbName);
  tableMeta.setTableName(catalogTable.getName());
  tableMeta.setTableType(catalogTable.getTableType());
  if (catalogTable.getParameters().containsKey("comment")) {
    tableMeta.setComments(catalogTable.getParameters().get("comment"));
  }
  return tableMeta;
}
 
Example #8
Source File: DatabaseMappingImplTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void transformOutboundTableMeta() throws Exception {
  TableMeta tableMeta = new TableMeta();
  tableMeta.setDbName(DB_NAME);
  TableMeta result = databaseMapping.transformOutboundTableMeta(tableMeta);
  assertThat(result, is(sameInstance(tableMeta)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 
Example #9
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<TableMeta> get_table_meta(String db_patterns, String tbl_patterns, List<String> tbl_types)
    throws MetaException, TException {
  return databaseMappingService.getPanopticOperationHandler()
      .getTableMeta(db_patterns, tbl_patterns, tbl_types);
}
 
Example #10
Source File: GetTableMetaRequest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public List<TableMeta> call() throws TException {
  List<TableMeta> tables = mapping.getClient().get_table_meta(dbPattern, tablePattern, tableTypes);
  List<TableMeta> mappedTableMeta = new ArrayList<>();
  for (TableMeta tableMeta : tables) {
    if (filter.apply(tableMeta, mapping)) {
      mappedTableMeta.add(mapping.transformOutboundTableMeta(tableMeta));
    }
  }
  return mappedTableMeta;
}
 
Example #11
Source File: GetTableMetaRequest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
public GetTableMetaRequest(
    DatabaseMapping mapping,
    String dbPattern,
    String tablePattern,
    List<String> tableTypes,
    BiFunction<TableMeta, DatabaseMapping, Boolean> filter) {
  this.mapping = mapping;
  this.dbPattern = dbPattern;
  this.tablePattern = tablePattern;
  this.tableTypes = tableTypes;
  this.filter = filter;
}
 
Example #12
Source File: GlueMetastoreClientDelegateTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetTableMeta() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = Lists.newArrayList(TableType.MANAGED_TABLE.name());

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
      new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
      new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 
Example #13
Source File: EntityConversionTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Test
public void testTableMetaConversion() {
  Table catalogTable = TestObjects.getTestTable();
  TableMeta tableMeta = CatalogToHiveConverter.convertTableMeta(catalogTable, TEST_DB_NAME);
  assertEquals(catalogTable.getName(), tableMeta.getTableName());
  assertEquals(TEST_DB_NAME, tableMeta.getDbName());
  assertEquals(catalogTable.getTableType(), tableMeta.getTableType());
}
 
Example #14
Source File: IdentityMapping.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
public TableMeta transformOutboundTableMeta(TableMeta tableMeta) {
  return tableMeta;
}
 
Example #15
Source File: DatabaseMappingImpl.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
public TableMeta transformOutboundTableMeta(TableMeta tableMeta) {
  tableMeta.setDbName(metaStoreMapping.transformOutboundDatabaseName(tableMeta.getDbName()));
  return tableMeta;
}
 
Example #16
Source File: IdentityMappingTest.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Test
public void transformOutboundTableMeta() throws Exception {
  TableMeta tableMeta = new TableMeta();
  TableMeta result = databaseMapping.transformOutboundTableMeta(tableMeta);
  assertThat(result, is(sameInstance(tableMeta)));
}
 
Example #17
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 4 votes vote down vote up
@Override
public List<TableMeta> getTableMeta(String dbPatterns, String tablePatterns, List<String> tableTypes)
    throws MetaException, TException, UnknownDBException {
  return glueMetastoreClientDelegate.getTableMeta(dbPatterns, tablePatterns, tableTypes);
}
 
Example #18
Source File: SubmarineMetaStore.java    From submarine with Apache License 2.0 4 votes vote down vote up
public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
    throws MetaException {
  List<TableMeta> tableMetas = rs.getTableMeta(dbNames, tableNames, tableTypes);
  return tableMetas;
}
 
Example #19
Source File: PanopticOperationHandler.java    From waggle-dance with Apache License 2.0 2 votes vote down vote up
/**
 * Implements {@link HMSHandler#get_table_meta(String, String, List)} over multiple metastores
 *
 * @param databasePatterns database patterns to match
 * @param tablePatterns table patterns to match
 * @param tableTypes table types to match
 * @return list of table metadata
 */
abstract public List<TableMeta> getTableMeta(String databasePatterns, String tablePatterns, List<String> tableTypes);
 
Example #20
Source File: DatabaseMapping.java    From waggle-dance with Apache License 2.0 votes vote down vote up
TableMeta transformOutboundTableMeta(TableMeta tableMeta);