org.apache.hadoop.hive.metastore.api.EnvironmentContext Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.EnvironmentContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HiveShimV210.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void alterPartition(IMetaStoreClient client, String databaseName, String tableName, Partition partition)
		throws InvalidOperationException, MetaException, TException {
	String errorMsg = "Failed to alter partition for table %s in database %s";
	try {
		Method method = client.getClass().getMethod("alter_partition", String.class, String.class,
			Partition.class, EnvironmentContext.class);
		method.invoke(client, databaseName, tableName, partition, null);
	} catch (InvocationTargetException ite) {
		Throwable targetEx = ite.getTargetException();
		if (targetEx instanceof TException) {
			throw (TException) targetEx;
		} else {
			throw new CatalogException(String.format(errorMsg, tableName, databaseName), targetEx);
		}
	} catch (NoSuchMethodException | IllegalAccessException e) {
		throw new CatalogException(String.format(errorMsg, tableName, databaseName), e);
	}
}
 
Example #2
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_fields_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_fields_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);

            if (table == null || table.getSd() == null || table.getSd().getCols() == null) {
                throw new MetaException("Unable to get fields for " + dbName + "." + tableName);
            }
            return table.getSd().getCols();
        });
}
 
Example #3
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void drop_table_with_environment_context(final String dbname, final String name,
                                                final boolean deleteData,
                                                @Nullable final EnvironmentContext ec) throws TException {
    requestWrapper("drop_table_with_environment_context", new Object[]{dbname, name, deleteData, ec}, () -> {
        final String databaseName = normalizeIdentifier(dbname);
        final String tableName = normalizeIdentifier(name);

        if (deleteData) {
            log.warn("Ignoring command to delete data for {}/{}/{}", catalogName, databaseName, tableName);
        }

        return v1.deleteTable(catalogName, databaseName, tableName);
    });
}
 
Example #4
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public boolean drop_partition_by_name_with_environment_context(
    final String dbName, final String tblName,
    final String partName,
    final boolean deleteData,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("drop_partition_by_name_with_environment_context",
        new Object[]{dbName, tblName, partName, deleteData, environmentContext}, () -> {
            final String databaseName = normalizeIdentifier(dbName);
            final String tableName = normalizeIdentifier(tblName);

            if (deleteData) {
                log.warn("Ignoring command to delete data for {}/{}/{}/{}",
                    catalogName, databaseName, tableName, partName);
            }

            partV1.deletePartitions(catalogName, databaseName, tableName, ImmutableList.of(partName));

            return true;
        });
}
 
Example #5
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void create_table_with_environment_context(
    final Table tbl,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    requestWrapper("create_table_with_environment_context", new Object[]{tbl, environmentContext}, () -> {
        final String dbname = normalizeIdentifier(tbl.getDbName());
        final String tblName = normalizeIdentifier(tbl.getTableName());
        final QualifiedName name = QualifiedName.ofTable(catalogName, dbname, tblName);

        final TableDto dto = hiveConverters.hiveToMetacatTable(name, tbl);
        v1.createTable(catalogName, dbname, tblName, dto);
        return null;
    });
}
 
Example #6
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public Partition append_partition_with_environment_context(
    final String dbName,
    final String tblName,
    final List<String> partVals,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("append_partition_by_name_with_environment_context",
        new Object[]{dbName, tblName, partVals}, () -> {
            final TableDto tableDto = getTableDto(dbName, tblName);
            final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals);
            appendPartitionsCore(dbName, tblName, partName);
            return hiveConverters.metacatToHivePartition(getPartitionDtoByName(tableDto, partName), tableDto);
        });
}
 
Example #7
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void alter_table_with_environment_context(
    final String dbname,
    final String tblName,
    final Table newTbl,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    requestWrapper("alter_table_with_environment_context",
        new Object[]{dbname, tblName, newTbl, environmentContext}, () -> {
            final String databaseName = normalizeIdentifier(dbname);
            final String tableName = normalizeIdentifier(tblName);
            final QualifiedName oldName = QualifiedName.ofTable(catalogName, databaseName, tableName);
            final QualifiedName newName = QualifiedName
                .ofTable(catalogName, newTbl.getDbName(), newTbl.getTableName());

            final TableDto dto = hiveConverters.hiveToMetacatTable(newName, newTbl);
            if (!oldName.equals(newName)) {
                v1.renameTable(catalogName, oldName.getDatabaseName(), oldName.getTableName(),
                    newName.getTableName());
            }
            v1.updateTable(catalogName, dbname, newName.getTableName(), dto);
            return null;
        });
}
 
Example #8
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void alter_partition_with_environment_context(
    final String dbName,
    final String tblName,
    final Partition newPart,
    @Nullable final EnvironmentContext ec
) throws TException {
    final String databaseName = normalizeIdentifier(dbName);
    final String tableName = normalizeIdentifier(tblName);
    requestWrapper("alter_partition_with_environment_context", new Object[]{databaseName, tableName, ec},
        () -> {
            addPartitionsCore(dbName, tableName, ImmutableList.of(newPart), false);
            return null;
        });
}
 
Example #9
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void create_table_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  Table table = new Table();
  table.setDbName(DB_P);
  Table inboundTable = new Table();
  inboundTable.setDbName("inbound");
  when(primaryMapping.transformInboundTable(table)).thenReturn(inboundTable);
  handler.create_table_with_environment_context(table, environmentContext);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).create_table_with_environment_context(inboundTable, environmentContext);
}
 
Example #10
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<FieldSchema> get_schema_with_environment_context(
    String db_name,
    String table_name,
    EnvironmentContext environment_context)
    throws MetaException, UnknownTableException, UnknownDBException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  return mapping
      .getClient()
      .get_schema_with_environment_context(mapping.transformInboundDatabaseName(db_name), table_name,
          environment_context);
}
 
Example #11
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void drop_table_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  handler.drop_table_with_environment_context(DB_P, "table", false, environmentContext);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).drop_table_with_environment_context("inbound", "table", false, environmentContext);
}
 
Example #12
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void alter_table_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  Table table = new Table();
  table.setDbName(DB_P);
  Table inbound = new Table();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryMapping.transformInboundTable(table)).thenReturn(inbound);
  handler.alter_table_with_environment_context(DB_P, "table", table, environmentContext);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_table_with_environment_context("inbound", "table", inbound, environmentContext);
}
 
Example #13
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void add_partition_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  Partition newPartition = new Partition();
  newPartition.setDbName(DB_P);
  Partition inbound = new Partition();
  Partition outbound = new Partition();
  when(primaryMapping.transformInboundPartition(newPartition)).thenReturn(inbound);
  when(primaryClient.add_partition_with_environment_context(inbound, environmentContext)).thenReturn(inbound);
  when(primaryMapping.transformOutboundPartition(inbound)).thenReturn(outbound);
  Partition result = handler.add_partition_with_environment_context(newPartition, environmentContext);
  assertThat(result, is(outbound));
  verify(primaryMapping).checkWritePermissions(DB_P);
}
 
Example #14
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void append_partition_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  Partition inbound = new Partition();
  Partition outbound = new Partition();
  List<String> partVals = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient.append_partition_with_environment_context("inbound", "table1", partVals, environmentContext))
      .thenReturn(inbound);
  when(primaryMapping.transformOutboundPartition(inbound)).thenReturn(outbound);
  Partition result = handler.append_partition_with_environment_context(DB_P, "table1", partVals, environmentContext);
  assertThat(result, is(outbound));
  verify(primaryMapping).checkWritePermissions(DB_P);
}
 
Example #15
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void append_partition_by_name_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  Partition inbound = new Partition();
  Partition outbound = new Partition();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient
      .append_partition_by_name_with_environment_context("inbound", "table1", "partName", environmentContext))
      .thenReturn(inbound);
  when(primaryMapping.transformOutboundPartition(inbound)).thenReturn(outbound);
  Partition result = handler
      .append_partition_by_name_with_environment_context(DB_P, "table1", "partName", environmentContext);
  assertThat(result, is(outbound));
  verify(primaryMapping).checkWritePermissions(DB_P);
}
 
Example #16
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void drop_partition_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  List<String> partVals = Lists.newArrayList();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(
      primaryClient.drop_partition_with_environment_context("inbound", "table1", partVals, false, environmentContext))
      .thenReturn(true);
  boolean result = handler
      .drop_partition_with_environment_context(DB_P, "table1", partVals, false, environmentContext);
  assertThat(result, is(true));
  verify(primaryMapping).checkWritePermissions(DB_P);
}
 
Example #17
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void drop_partition_by_name_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");
  when(primaryClient
      .drop_partition_by_name_with_environment_context("inbound", "table1", "partName", false, environmentContext))
      .thenReturn(true);
  boolean result = handler
      .drop_partition_by_name_with_environment_context(DB_P, "table1", "partName", false, environmentContext);
  assertThat(result, is(true));
  verify(primaryMapping).checkWritePermissions(DB_P);
}
 
Example #18
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void alter_partition_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  Partition newPartition = new Partition();
  newPartition.setDbName(DB_P);
  Partition inbound = new Partition();
  when(primaryMapping.transformInboundPartition(newPartition)).thenReturn(inbound);
  handler.alter_partition_with_environment_context(DB_P, "table", newPartition, environmentContext);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_partition_with_environment_context(DB_P, "table", inbound, environmentContext);
}
 
Example #19
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void alter_partitions_with_environment_context() throws TException {
  EnvironmentContext environmentContext = new EnvironmentContext();
  handler.alter_partitions_with_environment_context(DB_P, "table", Collections.emptyList(), environmentContext);
  verify(primaryMapping).checkWritePermissions(DB_P);
  verify(primaryClient).alter_partitions_with_environment_context(DB_P, "table", Collections.emptyList(),
      environmentContext);
}
 
Example #20
Source File: FederatedHMSHandlerTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void get_fields_with_environment_context() throws TException {
  EnvironmentContext context = new EnvironmentContext();
  List<FieldSchema> expected = Arrays.asList(new FieldSchema("name1", "type1", ""),
      new FieldSchema("name2", "type2", ""));
  when(primaryClient.get_fields_with_environment_context(DB_P, "table", context)).thenReturn(expected);
  List<FieldSchema> result = handler.get_fields_with_environment_context(DB_P, "table", context);
  assertThat(result, is(expected));
}
 
Example #21
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public Partition add_partition_with_environment_context(
    final Partition newPart,
    @Nullable final EnvironmentContext ec
) throws TException {
    final String dbName = normalizeIdentifier(newPart.getDbName());
    final String tableName = normalizeIdentifier(newPart.getTableName());
    return requestWrapper("add_partition_with_environment_context", new Object[]{dbName, tableName, ec}, () -> {
        addPartitionsCore(dbName, tableName, ImmutableList.of(newPart), false);
        return newPart;
    });
}
 
Example #22
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public Partition append_partition_by_name_with_environment_context(
    final String dbName, final String tblName,
    final String partName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("append_partition_by_name_with_environment_context",
        new Object[]{dbName, tblName, partName},
        () -> appendPartitionsCoreAndReturn(dbName, tblName, partName));
}
 
Example #23
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public boolean drop_partition_with_environment_context(
    final String dbName, final String tblName,
    final List<String> partVals,
    final boolean deleteData,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("drop_partition_with_environment_context",
        new Object[]{dbName, tblName, partVals, deleteData, environmentContext}, () -> {
            final TableDto tableDto = getTableDto(dbName, tblName);
            final String partName = hiveConverters.getNameFromPartVals(tableDto, partVals);

            final QualifiedName partitionName = getPartitionDtoByName(tableDto, partName).getName();

            if (deleteData) {
                log.warn("Ignoring command to delete data for {}/{}/{}/{}",
                    catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(),
                    partitionName.getPartitionName());
            }

            partV1.deletePartitions(
                catalogName, tableDto.getName().getDatabaseName(), tableDto.getName().getTableName(),
                ImmutableList.of(partitionName.getPartitionName()));

            return true;
        });
}
 
Example #24
Source File: CatalogThriftHiveMetastore.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public List<FieldSchema> get_schema_with_environment_context(
    final String dbName,
    final String tableName,
    @Nullable final EnvironmentContext environmentContext
) throws TException {
    return requestWrapper("get_schema_with_environment_context",
        new Object[]{dbName, tableName, environmentContext}, () -> {
            final Table table = get_table(dbName, tableName);
            List<FieldSchema> partitionKeys = Collections.emptyList();
            List<FieldSchema> columns = Collections.emptyList();

            if (table != null && table.getSd() != null && table.getSd().getCols() != null) {
                columns = table.getSd().getCols();
            }

            if (table != null && table.getPartitionKeys() != null) {
                partitionKeys = table.getPartitionKeys();
            }

            if (partitionKeys.isEmpty() && columns.isEmpty()) {
                throw new MetaException(
                    "Table does not have any partition keys or cols: " + dbName + "." + tableName);
            }

            final List<FieldSchema> result = Lists.newArrayListWithCapacity(partitionKeys.size() + columns.size());
            result.addAll(columns);
            result.addAll(partitionKeys);
            return result;
        });
}
 
Example #25
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public List<FieldSchema> get_fields_with_environment_context(
    String db_name,
    String table_name,
    EnvironmentContext environment_context)
    throws MetaException, UnknownTableException, UnknownDBException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(db_name);
  return mapping
      .getClient()
      .get_fields_with_environment_context(mapping.transformInboundDatabaseName(db_name), table_name,
          environment_context);
}
 
Example #26
Source File: AwsGlueHive2Shims.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public boolean requireCalStats(
    Configuration conf,
    Partition oldPart,
    Partition newPart,
    Table tbl,
    EnvironmentContext environmentContext) {
  return MetaStoreUtils.requireCalStats(conf, oldPart, newPart, tbl, environmentContext);
}
 
Example #27
Source File: AwsGlueHive2Shims.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public boolean updateTableStatsFast(
    Database db,
    Table tbl,
    Warehouse wh,
    boolean madeDir,
    boolean forceRecompute,
    EnvironmentContext environmentContext
) throws MetaException {
  return MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir, forceRecompute, environmentContext);
}
 
Example #28
Source File: AwsGlueSparkHiveShims.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public boolean requireCalStats(
    Configuration conf,
    Partition oldPart,
    Partition newPart,
    Table tbl,
    EnvironmentContext environmentContext) {
  return MetaStoreUtils.requireCalStats(conf, oldPart, newPart, tbl);
}
 
Example #29
Source File: AwsGlueSparkHiveShims.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public boolean updateTableStatsFast(
    Database db,
    Table tbl,
    Warehouse wh,
    boolean madeDir,
    boolean forceRecompute,
    EnvironmentContext environmentContext
) throws MetaException {
  return MetaStoreUtils.updateUnpartitionedTableStatsFast(db, tbl, wh, madeDir, forceRecompute);
}
 
Example #30
Source File: AWSCatalogMetastoreClient.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Override
public void alter_partition(
    String dbName,
    String tblName,
    org.apache.hadoop.hive.metastore.api.Partition partition,
    EnvironmentContext environmentContext
) throws InvalidOperationException, MetaException, TException {
  glueMetastoreClientDelegate.alterPartitions(dbName, tblName, Lists.newArrayList(partition));
}