org.apache.hadoop.hive.metastore.api.MetaException Java Examples

@Override
protected HiveMetaStoreClient newClient()  {
  try {
    return new HiveMetaStoreClient(hiveConf);
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
  } catch (Throwable t) {
    if (t.getMessage().contains("Another instance of Derby may have already booted")) {
      throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " +
          "Derby supports only one client at a time. To fix this, use a metastore that supports " +
          "multiple clients.");
    }

    throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore");
  }
}
 

@Override
public void preCreateTable(Table table) throws MetaException {
  DynamoDBClient client = createDynamoDBClient(table);
  try {

    boolean isExternal = MetaStoreUtils.isExternalTable(table);

    if (!isExternal) {
      throw new MetaException("Only EXTERNAL tables are supported for DynamoDB.");
    }

    String tableName = HiveDynamoDBUtil.getDynamoDBTableName(table.getParameters()
        .get(DynamoDBConstants.TABLE_NAME), table.getTableName());
    TableDescription tableDescription = client.describeTable(tableName);

    checkTableStatus(tableDescription);
    checkTableSchemaMapping(tableDescription, table);
    checkTableSchemaType(tableDescription, table);
  } finally {
    client.close();
  }
}
 

public static String buildExpressionFromPartialSpecification(org.apache.hadoop.hive.metastore.api.Table table,
        List<String> partitionValues) throws MetaException {

  List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys = table.getPartitionKeys();

  if (partitionValues == null || partitionValues.isEmpty() ) {
    return null;
  }

  if (partitionKeys == null || partitionValues.size() > partitionKeys.size()) {
    throw new MetaException("Incorrect number of partition values: " + partitionValues);
  }

  partitionKeys = partitionKeys.subList(0, partitionValues.size());
  List<String> predicates = new LinkedList<>();
  for (int i = 0; i < partitionValues.size(); i++) {
    if (!Strings.isNullOrEmpty(partitionValues.get(i))) {
      predicates.add(buildPredicate(partitionKeys.get(i), partitionValues.get(i)));
    }
  }

  return JOINER.join(predicates);
}
 

private void performDropPartitionPostProcessing(String dbName, String tblName,
                                                org.apache.hadoop.hive.metastore.api.Partition partition, boolean deleteData, boolean ifPurge)
      throws MetaException, NoSuchObjectException, TException {
    if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
        Path partPath = new Path(partition.getSd().getLocation());
        org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
        if (isExternalTable(table)){
            //Don't delete external table data
            return;
        }
        boolean mustPurge = isMustPurge(table, ifPurge);
        wh.deleteDir(partPath, true, mustPurge);
        try {
            List<String> values = partition.getValues();
            deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
        } catch (IOException e) {
            throw new MetaException(e.getMessage());
        }
    }
}
 

@Override
public TProcessor getProcessor(TTransport transport) {
  try {
    if (transport instanceof TSocket) {
      Socket socket = ((TSocket) transport).getSocket();
      log.debug("Received a connection from ip: {}", socket.getInetAddress().getHostAddress());
    }
    CloseableIHMSHandler baseHandler = federatedHMSHandlerFactory.create();
    IHMSHandler handler = newRetryingHMSHandler(ExceptionWrappingHMSHandler.newProxyInstance(baseHandler), hiveConf,
        false);
    transportMonitor.monitor(transport, baseHandler);
    return new TSetIpAddressProcessor<>(handler);
  } catch (MetaException | ReflectiveOperationException | RuntimeException e) {
    throw new RuntimeException("Error creating TProcessor", e);
  }
}
 

@Override
public void onCreateDatabase(CreateDatabaseEvent dbEvent)
    throws MetaException {

  // don't sync paths/privileges if the operation has failed
  if (!dbEvent.getStatus()) {
    LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
      " since the operation failed. \n");
    return;
  }

  if (dbEvent.getDatabase().getLocationUri() != null) {
    String authzObj = dbEvent.getDatabase().getName();
    String path = dbEvent.getDatabase().getLocationUri();
    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
      plugin.addPath(authzObj, path);
    }
  }
  // drop the privileges on the database, in case anything left behind during
  // last drop db
  if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
    return;
  }

  dropSentryDbPrivileges(dbEvent.getDatabase().getName());
}
 

/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
 

private void performDropPartitionPostProcessing(
    String dbName,
    String tblName,
    org.apache.hadoop.hive.metastore.api.Partition partition,
    boolean deleteData,
    boolean ifPurge
) throws TException {
  if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
    Path partPath = new Path(partition.getSd().getLocation());
    org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
    if (isExternalTable(table)) {
      //Don't delete external table data
      return;
    }
    boolean mustPurge = isMustPurge(table, ifPurge);
    wh.deleteDir(partPath, true, mustPurge);
    try {
      List<String> values = partition.getValues();
      deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
    } catch (IOException e) {
      throw new MetaException(e.getMessage());
    }
  }
}
 

@Test
public void testCheckTableSchemaNullSerializationValid() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
      "col2:dynamo_col2#,hashKey:hashKey");
  parameters.put(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, "true");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "array<bigint>", ""));
  cols.add(new FieldSchema("hashKey", "string", ""));
  sd.setCols(cols);
  table.setSd(sd);

  // This check is expected to pass for the given input
  storageHandler.checkTableSchemaType(description, table);
}
 

private HiveStorageFormat extractHiveStorageFormat(final Table table) throws MetaException {
    final StorageDescriptor descriptor = table.getSd();
    if (descriptor == null) {
        throw new MetaException("Table is missing storage descriptor");
    }
    final SerDeInfo serdeInfo = descriptor.getSerdeInfo();
    if (serdeInfo == null) {
        throw new MetaException(
            "Table storage descriptor is missing SerDe info");
    }
    final String outputFormat = descriptor.getOutputFormat();
    final String serializationLib = serdeInfo.getSerializationLib();

    for (HiveStorageFormat format : HiveStorageFormat.values()) {
        if (format.getOutputFormat().equals(outputFormat) && format.getSerde().equals(serializationLib)) {
            return format;
        }
    }
    throw new MetaException(
        String.format("Output format %s with SerDe %s is not supported", outputFormat, serializationLib));
}
 

private synchronized void setProxiedConnection(final List<String> proxies)
    throws IOException, InterruptedException, TException {
  Preconditions.checkArgument(this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
      "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
  String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM);
  UserGroupInformation loginUser = UserGroupInformation
      .loginUserFromKeytabAndReturnUGI(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation);
  loginUser.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run()
        throws MetaException, SQLException, ClassNotFoundException {
      for (String proxy : proxies) {
        HiveConnection hiveConnection = getHiveConnection(Optional.fromNullable(proxy));
        Statement statement = hiveConnection.createStatement();
        statementMap.put(proxy, statement);
        connectionMap.put(proxy, hiveConnection);
        for (String setting : settings) {
          statement.execute(setting);
        }
      }
      return null;
    }
  });
}
 

@Override
public void dropConstraint(
    String dbName,
    String tblName,
    String constraintName
) throws MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.dropConstraint(dbName, tblName, constraintName);
}
 

private void reloginExpiringKeytabUser() throws MetaException {
  if(UserGroupInformation.isSecurityEnabled()) {
    // renew the TGT if required
    try {
      UserGroupInformation ugi = UserGroupInformation.getLoginUser();
      if (ugi.isFromKeytab()) {
        ugi.checkTGTAndReloginFromKeytab();
      }
    } catch (IOException e) {
      final String msg = "Error doing relogin using keytab " + e.getMessage();
      logger.error(msg, e);
      throw new MetaException(msg);
    }
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AggrStats get_aggr_stats_for(PartitionsStatsRequest request)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(request.getDbName());
  return mapping.getClient().get_aggr_stats_for(mapping.transformInboundPartitionsStatsRequest(request));
}
 

@Override
public List<org.apache.hadoop.hive.metastore.api.Table> getTableObjectsByName(String dbName, List<String> tableNames) throws MetaException,
      InvalidOperationException, UnknownDBException, TException {
    List<org.apache.hadoop.hive.metastore.api.Table> hiveTables = Lists.newArrayList();
    for(String tableName : tableNames) {
        hiveTables.add(getTable(dbName, tableName));
    }

    return hiveTables;
}
 

/**
 * Create the column family if it doesn't exist.
 * @param ks
 * @return
 * @throws MetaException
 */
public CfDef createCFIfNotFound(KsDef ks) throws MetaException {
  CfDef cf = getColumnFamily(ks);
  if (cf == null) {
    return createColumnFamily();
  } else {
    return cf;
  }
}
 

@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> dropPartitions(
    String dbName,
    String tblName,
    List<ObjectPair<Integer, byte[]>> partExprs,
    boolean deleteData,
    boolean ifExists,
    boolean needResults
) throws NoSuchObjectException, MetaException, TException {
  return dropPartitions_core(dbName, tblName, partExprs, deleteData, false);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public GetTableResult get_table_req(GetTableRequest req) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(req.getDbName());
  GetTableResult result = mapping.getClient().get_table_req(mapping.transformInboundGetTableRequest(req));
  return mapping.transformOutboundGetTableResult(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Table get_table(String dbname, String tbl_name) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(dbname);
  return mapping
      .transformOutboundTable(mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name));
}
 

@Override
public int add_partitions_pspec(
    PartitionSpecProxy pSpec
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException,
    MetaException, TException {
  return glueMetastoreClientDelegate.addPartitionsSpecProxy(pSpec);
}
 

/**
 * Constructor.
 *
 * @param address                    address
 * @param hiveMetastoreClientFactory hiveMetastoreClientFactory
 * @throws MetaException exception
 */
public MetacatHiveClient(final URI address,
                         final HiveMetastoreClientFactory hiveMetastoreClientFactory)
        throws MetaException {
    this.hiveMetastoreClientFactory = hiveMetastoreClientFactory;
    Preconditions.checkArgument(address.getHost() != null, "metastoreUri host is missing: " + address);
    Preconditions.checkArgument(address.getPort() != -1, "metastoreUri port is missing: " + address);
    this.host = address.getHost();
    this.port = address.getPort();
}
 

@Test(expected = MetaException.class)
public void testGetPartitionsPartialFailure() throws Exception {
  List<String> partitionKeys1 = Arrays.asList("foo1", "bar1");
  final Partition partition1 = new Partition().withDatabaseName(testDb.getName())
          .withTableName(testTbl.getName())
          .withValues(partitionKeys1);

  when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
          .thenAnswer(new Answer<GetPartitionsResult>() {
            @Override
            public GetPartitionsResult answer(InvocationOnMock invocation) {
              GetPartitionsRequest request = invocation.getArgumentAt(0, GetPartitionsRequest.class);
              GetPartitionsResult result;
              switch (request.getSegment().getSegmentNumber()) {
                case 0:
                  result = new GetPartitionsResult().withPartitions(Lists.newArrayList(partition1));
                  break;
                default:
                  throw new OperationTimeoutException("timeout");
              }
              return result;
            }
          });

  List<org.apache.hadoop.hive.metastore.api.Partition> res = metastoreClientDelegate.getPartitions(
          testDb.getName(), testTbl.getName(), null, -1);
}
 

/**
 * Verify alter table privileges
 * @throws Exception
 */
@Test
public void testAlterTablePrivileges() throws Exception {

  HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
  createMetastoreTable(client, dbName, tabName1,
      Lists.newArrayList(new FieldSchema("col1", "int", "")));
  client.close();

  // verify group1 users with DDL privileges can alter tables in db_1
  client = context.getMetaStoreClient(USER1_1);
  Table metaTable2 = client.getTable(dbName, tabName1);
  metaTable2.getSd().setCols(
      Lists.newArrayList(new FieldSchema("col2", "double", "")));
  client.alter_table(dbName, tabName1, metaTable2);
  Table metaTable3 = client.getTable(dbName, tabName1);
  assertEquals(metaTable2.getSd().getCols(), metaTable3.getSd().getCols());

  // verify group1 users with DDL privileges can alter tables in db_1
  client = context.getMetaStoreClient(USER2_1);
  metaTable2 = client.getTable(dbName, tabName1);
  metaTable2.getSd().setCols(
      Lists.newArrayList(new FieldSchema("col3", "string", "")));
  client.alter_table(dbName, tabName1, metaTable2);
  metaTable3 = client.getTable(dbName, tabName1);
  assertEquals(metaTable2.getSd().getCols(), metaTable3.getSd().getCols());

  // verify group3 users can't alter tables in db_1
  client = context.getMetaStoreClient(USER3_1);
  metaTable2 = client.getTable(dbName, tabName1);
  metaTable2.getSd().setCols(
      Lists.newArrayList(new FieldSchema("col3", "string", "")));
  try {
    client.alter_table(dbName, tabName1, metaTable2);
    fail("alter table should have failed for non-privilege user");
  } catch (MetaException e) {
    Context.verifyMetastoreAuthException(e);
  }
  client.close();
}
 

@Override
public PrimaryKeysResponse get_primary_keys(PrimaryKeysRequest request)
  throws MetaException, NoSuchObjectException, TException {
  // making sure the table exists
  client.get_table(request.getDb_name(), request.getTbl_name());
  // get_primary_keys is not supported in hive < 2.1 so just returning empty list.
  return new PrimaryKeysResponse(Collections.emptyList());
}
 

@Override
public AWSGlue newClient() throws MetaException {
  try {
    AWSGlueClientBuilder glueClientBuilder = AWSGlueClientBuilder.standard()
        .withCredentials(getAWSCredentialsProvider(conf));

    String regionStr = getProperty(AWS_REGION, conf);
    String glueEndpoint = getProperty(AWS_GLUE_ENDPOINT, conf);

    // ClientBuilder only allows one of EndpointConfiguration or Region to be set
    if (StringUtils.isNotBlank(glueEndpoint)) {
      logger.info("Setting glue service endpoint to " + glueEndpoint);
      glueClientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(glueEndpoint, null));
    } else if (StringUtils.isNotBlank(regionStr)) {
      logger.info("Setting region to : " + regionStr);
      glueClientBuilder.setRegion(regionStr);
    } else {
      Region currentRegion = Regions.getCurrentRegion();
      if (currentRegion != null) {
        logger.info("Using region from ec2 metadata : " + currentRegion.getName());
        glueClientBuilder.setRegion(currentRegion.getName());
      } else {
        logger.info("No region info found, using SDK default region: us-east-1");
      }
    }

    glueClientBuilder.setClientConfiguration(buildClientConfiguration(conf));
    return glueClientBuilder.build();
  } catch (Exception e) {
    String message = "Unable to build AWSGlueClient: " + e;
    logger.error(message);
    throw new MetaException(message);
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(req.getDbName());
  PartitionsByExprResult result = mapping
      .getClient()
      .get_partitions_by_expr(mapping.transformInboundPartitionsByExprRequest(req));
  return mapping.transformOutboundPartitionsByExprResult(result);
}
 

@Override
public Partition getPartition(String dbName, String tableName,
    List<String> part_vals) throws MetaException, NoSuchObjectException {
  if (filterTables(dbName, Lists.newArrayList(tableName)).isEmpty()) {
    throw new NoSuchObjectException(getNoAccessMessageForTable(dbName, tableName));
  }
  return super.getPartition(dbName, tableName, part_vals);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void markPartitionForEvent(
    String db_name,
    String tbl_name,
    Map<String, String> part_vals,
    PartitionEventType eventType)
    throws MetaException, NoSuchObjectException, UnknownDBException, UnknownTableException, UnknownPartitionException,
    InvalidPartitionException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  mapping
      .getClient()
      .markPartitionForEvent(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals, eventType);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean set_aggr_stats_for(SetPartitionsStatsRequest request)
    throws NoSuchObjectException, InvalidObjectException, MetaException, InvalidInputException, TException {
  if (!request.getColStats().isEmpty()) {
    DatabaseMapping mapping = databaseMappingService
        .databaseMapping(request.getColStats().get(0).getStatsDesc().getDbName());
    for (ColumnStatistics stats : request.getColStats()) {
      mapping.checkWritePermissions(stats.getStatsDesc().getDbName());
    }
    return mapping.getClient().set_aggr_stats_for(mapping.transformInboundSetPartitionStatsRequest(request));
  }
  return false;
}
 

@Override
void reconnect() throws MetaException {
  if (needDelegationToken) {
    getAndSetDelegationToken(hiveConf, ugiForRpc, processUserClient);
  }
  doAsCommand(
    (PrivilegedExceptionAction<Void>) () -> {
      client.reconnect();
      return null;
    },
      ugiForRpc,
      "Failed to reconnect to Hive metastore"
  );
}