Java Code Examples for org.apache.hadoop.hive.metastore.api.TableMeta

The following examples show how to use org.apache.hadoop.hive.metastore.api.TableMeta. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public List<TableMeta> getTableMeta(
    String dbPatterns,
    String tablePatterns,
    List<String> tableTypes
) throws TException  {
  List<TableMeta> tables = new ArrayList<>();
  List<String> databases = getDatabases(dbPatterns);
  for (String dbName : databases) {
    String nextToken = null;
    List<Table> dbTables = glueMetastore.getTables(dbName, tablePatterns);
    for (Table catalogTable : dbTables) {
      if (tableTypes == null ||
          tableTypes.isEmpty() ||
          tableTypes.contains(catalogTable.getTableType())) {
        tables.add(CatalogToHiveConverter.convertTableMeta(catalogTable, dbName));
      }
    }
  }
  return tables;
}
 
Example 2
@Test
public void testGetTableMetaNullEmptyTableType() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = null;

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
    new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
    new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));

  tableTypes = Lists.newArrayList();
  tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 
Example 3
Source Project: waggle-dance   Source File: PanopticOperationHandler.java    License: Apache License 2.0 6 votes vote down vote up
protected List<TableMeta> getTableMeta(
    String tablePatterns,
    List<String> tableTypes,
    Map<DatabaseMapping, String> databaseMappingsForPattern,
    BiFunction<TableMeta, DatabaseMapping, Boolean> filter) {
  List<GetTableMetaRequest> allRequests = new ArrayList<>();

  for (Entry<DatabaseMapping, String> mappingWithPattern : databaseMappingsForPattern.entrySet()) {
    DatabaseMapping mapping = mappingWithPattern.getKey();
    GetTableMetaRequest tableMetaRequest = new GetTableMetaRequest(mapping, mappingWithPattern.getValue(),
        tablePatterns, tableTypes, filter);
    allRequests.add(tableMetaRequest);
  }

  List<TableMeta> result = getPanopticOperationExecutor()
      .executeRequests(allRequests, GET_TABLE_META_TIMEOUT, "Got exception fetching get_table_meta: {}");
  return result;
}
 
Example 4
@Test
public void panopticOperationsHandlerGetTableMeta() throws Exception {
  TableMeta federatedTableMeta = new TableMeta("federated_db", "tbl", null);
  TableMeta primaryTableMeta = new TableMeta("primary_db", "tbl", null);

  when(primaryDatabaseClient.get_table_meta("*_db", "*", null))
      .thenReturn(Collections.singletonList(primaryTableMeta));
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta("*_db", "*", null))
      .thenReturn(Collections.singletonList(federatedTableMeta));
  when(metaStoreMappingFederated.transformOutboundDatabaseName("federated_db")).thenReturn("name_federated_db");

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> expected = Arrays.asList(primaryTableMeta, federatedTableMeta);
  List<TableMeta> result = handler.getTableMeta("*_db", "*", null);
  assertThat(result, is(expected));
}
 
Example 5
@Test
public void panopticOperationsHandlerGetTableMetaWithNonWhitelistedDb() throws TException {
  List<String> tblTypes = Lists.newArrayList();
  TableMeta tableMeta = new TableMeta("federated_db", "tbl", null);
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta("federated_*", "*", tblTypes))
      .thenReturn(Lists.newArrayList(tableMeta));

  // set metastore whitelist to be nonempty
  federatedMetastore.setMappedDatabases(Collections.singletonList("testName"));
  service = new PrefixBasedDatabaseMappingService(metaStoreMappingFactory,
      Arrays.asList(primaryMetastore, federatedMetastore), queryMapping);

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> tableMetas = handler.getTableMeta("name_federated_*", "*", tblTypes);

  assertThat(tableMetas.size(), is(0));
}
 
Example 6
@Test
public void panopticOperationsHandlerGetTableMeta() throws Exception {
  String pattern = "pattern";
  TableMeta primaryTableMeta = new TableMeta(PRIMARY_DB, "tbl", null);
  TableMeta federatedTableMeta = new TableMeta(FEDERATED_DB, "tbl", null);
  TableMeta ignoredTableMeta = new TableMeta("non_mapped_db", "tbl", null);

  when(primaryDatabaseClient.get_table_meta(pattern, pattern, null))
      .thenReturn(Collections.singletonList(primaryTableMeta));
  when(metaStoreMappingFederated.getClient()).thenReturn(federatedDatabaseClient);
  when(federatedDatabaseClient.get_table_meta(pattern, pattern, null))
      .thenReturn(Arrays.asList(federatedTableMeta, ignoredTableMeta));

  PanopticOperationHandler handler = service.getPanopticOperationHandler();
  List<TableMeta> expected = Lists.newArrayList(primaryTableMeta, federatedTableMeta);
  List<TableMeta> result = handler.getTableMeta(pattern, pattern, null);
  assertThat(result, is(expected));
}
 
Example 7
public static TableMeta convertTableMeta(com.amazonaws.services.glue.model.Table catalogTable, String dbName) {
  TableMeta tableMeta = new TableMeta();
  tableMeta.setDbName(dbName);
  tableMeta.setTableName(catalogTable.getName());
  tableMeta.setTableType(catalogTable.getTableType());
  if (catalogTable.getParameters().containsKey("comment")) {
    tableMeta.setComments(catalogTable.getParameters().get("comment"));
  }
  return tableMeta;
}
 
Example 8
@Test
public void testTableMetaConversion() {
  Table catalogTable = TestObjects.getTestTable();
  TableMeta tableMeta = CatalogToHiveConverter.convertTableMeta(catalogTable, TEST_DB_NAME);
  assertEquals(catalogTable.getName(), tableMeta.getTableName());
  assertEquals(TEST_DB_NAME, tableMeta.getDbName());
  assertEquals(catalogTable.getTableType(), tableMeta.getTableType());
}
 
Example 9
@Test
public void testGetTableMeta() throws Exception {
  List<Table> tables = Lists.newArrayList(testTbl);
  List<String> tableTypes = Lists.newArrayList(TableType.MANAGED_TABLE.name());

  when(glueClient.getDatabases(any(GetDatabasesRequest.class))).thenReturn(
      new GetDatabasesResult().withDatabaseList(testDb));
  when(glueClient.getTables(any(GetTablesRequest.class))).thenReturn(
      new GetTablesResult().withTableList(tables));

  List<TableMeta> tableMetaResult = metastoreClientDelegate.getTableMeta(testDb.getName(), testTbl.getName(), tableTypes);
  assertEquals(CatalogToHiveConverter.convertTableMeta(testTbl, testDb.getName()), Iterables.getOnlyElement(tableMetaResult));
}
 
Example 10
Source Project: waggle-dance   Source File: GetTableMetaRequest.java    License: Apache License 2.0 5 votes vote down vote up
public GetTableMetaRequest(
    DatabaseMapping mapping,
    String dbPattern,
    String tablePattern,
    List<String> tableTypes,
    BiFunction<TableMeta, DatabaseMapping, Boolean> filter) {
  this.mapping = mapping;
  this.dbPattern = dbPattern;
  this.tablePattern = tablePattern;
  this.tableTypes = tableTypes;
  this.filter = filter;
}
 
Example 11
Source Project: waggle-dance   Source File: GetTableMetaRequest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<TableMeta> call() throws TException {
  List<TableMeta> tables = mapping.getClient().get_table_meta(dbPattern, tablePattern, tableTypes);
  List<TableMeta> mappedTableMeta = new ArrayList<>();
  for (TableMeta tableMeta : tables) {
    if (filter.apply(tableMeta, mapping)) {
      mappedTableMeta.add(mapping.transformOutboundTableMeta(tableMeta));
    }
  }
  return mappedTableMeta;
}
 
Example 12
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<TableMeta> get_table_meta(String db_patterns, String tbl_patterns, List<String> tbl_types)
    throws MetaException, TException {
  return databaseMappingService.getPanopticOperationHandler()
      .getTableMeta(db_patterns, tbl_patterns, tbl_types);
}
 
Example 13
Source Project: waggle-dance   Source File: DatabaseMappingImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void transformOutboundTableMeta() throws Exception {
  TableMeta tableMeta = new TableMeta();
  tableMeta.setDbName(DB_NAME);
  TableMeta result = databaseMapping.transformOutboundTableMeta(tableMeta);
  assertThat(result, is(sameInstance(tableMeta)));
  assertThat(result.getDbName(), is(OUT_DB_NAME));
}
 
Example 14
Source Project: submarine   Source File: SubmarineMetaStore.java    License: Apache License 2.0 4 votes vote down vote up
public List<TableMeta> getTableMeta(String dbNames, String tableNames, List<String> tableTypes)
    throws MetaException {
  List<TableMeta> tableMetas = rs.getTableMeta(dbNames, tableNames, tableTypes);
  return tableMetas;
}
 
Example 15
@Override
public List<TableMeta> getTableMeta(String dbPatterns, String tablePatterns, List<String> tableTypes)
    throws MetaException, TException, UnknownDBException {
  return glueMetastoreClientDelegate.getTableMeta(dbPatterns, tablePatterns, tableTypes);
}
 
Example 16
Source Project: waggle-dance   Source File: DatabaseMappingImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TableMeta transformOutboundTableMeta(TableMeta tableMeta) {
  tableMeta.setDbName(metaStoreMapping.transformOutboundDatabaseName(tableMeta.getDbName()));
  return tableMeta;
}
 
Example 17
Source Project: waggle-dance   Source File: IdentityMapping.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TableMeta transformOutboundTableMeta(TableMeta tableMeta) {
  return tableMeta;
}
 
Example 18
Source Project: waggle-dance   Source File: IdentityMappingTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void transformOutboundTableMeta() throws Exception {
  TableMeta tableMeta = new TableMeta();
  TableMeta result = databaseMapping.transformOutboundTableMeta(tableMeta);
  assertThat(result, is(sameInstance(tableMeta)));
}
 
Example 19
Source Project: waggle-dance   Source File: PanopticOperationHandler.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Implements {@link HMSHandler#get_table_meta(String, String, List)} over multiple metastores
 *
 * @param databasePatterns database patterns to match
 * @param tablePatterns table patterns to match
 * @param tableTypes table types to match
 * @return list of table metadata
 */
abstract public List<TableMeta> getTableMeta(String databasePatterns, String tablePatterns, List<String> tableTypes);
 
Example 20
Source Project: waggle-dance   Source File: DatabaseMapping.java    License: Apache License 2.0 votes vote down vote up
TableMeta transformOutboundTableMeta(TableMeta tableMeta);