org.apache.hadoop.hive.metastore.api.MetaException Java Examples

private synchronized void setProxiedConnection(final List<String> proxies)
    throws IOException, InterruptedException, TException {
  Preconditions.checkArgument(this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
      "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
  String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM);
  UserGroupInformation loginUser = UserGroupInformation
      .loginUserFromKeytabAndReturnUGI(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation);
  loginUser.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run()
        throws MetaException, SQLException, ClassNotFoundException {
      for (String proxy : proxies) {
        HiveConnection hiveConnection = getHiveConnection(Optional.fromNullable(proxy));
        Statement statement = hiveConnection.createStatement();
        statementMap.put(proxy, statement);
        connectionMap.put(proxy, hiveConnection);
        for (String setting : settings) {
          statement.execute(setting);
        }
      }
      return null;
    }
  });
}
 

private HiveStorageFormat extractHiveStorageFormat(final Table table) throws MetaException {
    final StorageDescriptor descriptor = table.getSd();
    if (descriptor == null) {
        throw new MetaException("Table is missing storage descriptor");
    }
    final SerDeInfo serdeInfo = descriptor.getSerdeInfo();
    if (serdeInfo == null) {
        throw new MetaException(
            "Table storage descriptor is missing SerDe info");
    }
    final String outputFormat = descriptor.getOutputFormat();
    final String serializationLib = serdeInfo.getSerializationLib();

    for (HiveStorageFormat format : HiveStorageFormat.values()) {
        if (format.getOutputFormat().equals(outputFormat) && format.getSerde().equals(serializationLib)) {
            return format;
        }
    }
    throw new MetaException(
        String.format("Output format %s with SerDe %s is not supported", outputFormat, serializationLib));
}
 

public static String buildExpressionFromPartialSpecification(org.apache.hadoop.hive.metastore.api.Table table,
        List<String> partitionValues) throws MetaException {

  List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys = table.getPartitionKeys();

  if (partitionValues == null || partitionValues.isEmpty() ) {
    return null;
  }

  if (partitionKeys == null || partitionValues.size() > partitionKeys.size()) {
    throw new MetaException("Incorrect number of partition values: " + partitionValues);
  }

  partitionKeys = partitionKeys.subList(0, partitionValues.size());
  List<String> predicates = new LinkedList<>();
  for (int i = 0; i < partitionValues.size(); i++) {
    if (!Strings.isNullOrEmpty(partitionValues.get(i))) {
      predicates.add(buildPredicate(partitionKeys.get(i), partitionValues.get(i)));
    }
  }

  return JOINER.join(predicates);
}
 

private void performDropPartitionPostProcessing(String dbName, String tblName,
                                                org.apache.hadoop.hive.metastore.api.Partition partition, boolean deleteData, boolean ifPurge)
      throws MetaException, NoSuchObjectException, TException {
    if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
        Path partPath = new Path(partition.getSd().getLocation());
        org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
        if (isExternalTable(table)){
            //Don't delete external table data
            return;
        }
        boolean mustPurge = isMustPurge(table, ifPurge);
        wh.deleteDir(partPath, true, mustPurge);
        try {
            List<String> values = partition.getValues();
            deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
        } catch (IOException e) {
            throw new MetaException(e.getMessage());
        }
    }
}
 

@Override
public void onCreateDatabase(CreateDatabaseEvent dbEvent)
    throws MetaException {

  // don't sync paths/privileges if the operation has failed
  if (!dbEvent.getStatus()) {
    LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
      " since the operation failed. \n");
    return;
  }

  if (dbEvent.getDatabase().getLocationUri() != null) {
    String authzObj = dbEvent.getDatabase().getName();
    String path = dbEvent.getDatabase().getLocationUri();
    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
      plugin.addPath(authzObj, path);
    }
  }
  // drop the privileges on the database, in case anything left behind during
  // last drop db
  if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
    return;
  }

  dropSentryDbPrivileges(dbEvent.getDatabase().getName());
}
 

/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
 

private void performDropPartitionPostProcessing(
    String dbName,
    String tblName,
    org.apache.hadoop.hive.metastore.api.Partition partition,
    boolean deleteData,
    boolean ifPurge
) throws TException {
  if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
    Path partPath = new Path(partition.getSd().getLocation());
    org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
    if (isExternalTable(table)) {
      //Don't delete external table data
      return;
    }
    boolean mustPurge = isMustPurge(table, ifPurge);
    wh.deleteDir(partPath, true, mustPurge);
    try {
      List<String> values = partition.getValues();
      deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
    } catch (IOException e) {
      throw new MetaException(e.getMessage());
    }
  }
}
 

@Test
public void testCheckTableSchemaNullSerializationValid() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
      "col2:dynamo_col2#,hashKey:hashKey");
  parameters.put(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, "true");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "array<bigint>", ""));
  cols.add(new FieldSchema("hashKey", "string", ""));
  sd.setCols(cols);
  table.setSd(sd);

  // This check is expected to pass for the given input
  storageHandler.checkTableSchemaType(description, table);
}
 

@Override
public void preCreateTable(Table table) throws MetaException {
  DynamoDBClient client = createDynamoDBClient(table);
  try {

    boolean isExternal = MetaStoreUtils.isExternalTable(table);

    if (!isExternal) {
      throw new MetaException("Only EXTERNAL tables are supported for DynamoDB.");
    }

    String tableName = HiveDynamoDBUtil.getDynamoDBTableName(table.getParameters()
        .get(DynamoDBConstants.TABLE_NAME), table.getTableName());
    TableDescription tableDescription = client.describeTable(tableName);

    checkTableStatus(tableDescription);
    checkTableSchemaMapping(tableDescription, table);
    checkTableSchemaType(tableDescription, table);
  } finally {
    client.close();
  }
}
 

@Override
protected HiveMetaStoreClient newClient()  {
  try {
    return new HiveMetaStoreClient(hiveConf);
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
  } catch (Throwable t) {
    if (t.getMessage().contains("Another instance of Derby may have already booted")) {
      throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " +
          "Derby supports only one client at a time. To fix this, use a metastore that supports " +
          "multiple clients.");
    }

    throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore");
  }
}
 

@Override
public TProcessor getProcessor(TTransport transport) {
  try {
    if (transport instanceof TSocket) {
      Socket socket = ((TSocket) transport).getSocket();
      log.debug("Received a connection from ip: {}", socket.getInetAddress().getHostAddress());
    }
    CloseableIHMSHandler baseHandler = federatedHMSHandlerFactory.create();
    IHMSHandler handler = newRetryingHMSHandler(ExceptionWrappingHMSHandler.newProxyInstance(baseHandler), hiveConf,
        false);
    transportMonitor.monitor(transport, baseHandler);
    return new TSetIpAddressProcessor<>(handler);
  } catch (MetaException | ReflectiveOperationException | RuntimeException e) {
    throw new RuntimeException("Error creating TProcessor", e);
  }
}
 

@Override
public org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet get_privilege_set(
    HiveObjectRef obj,
    String user, List<String> groups
) throws MetaException, TException {
  return glueMetastoreClientDelegate.getPrivilegeSet(obj, user, groups);
}
 

@Override
void connect() throws MetaException {
  doAsCommand(
    (PrivilegedExceptionAction<Void>) () -> {
      client = Hive.get(hiveConf).getMSC();
      return null;
    },
      ugiForRpc,
      "Failed to connect to Hive metastore"
  );

  // Hive authorization helper needs the query user
  // (not the user from metastore client used to communicate with metastore)
  this.authorizer = new HiveAuthorizationHelper(client, hiveConf, userName);
}
 

@Override
public org.apache.hadoop.hive.metastore.api.Partition appendPartition(
    String dbName,
    String tblName,
    List<String> values
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException, MetaException, TException {
  return glueMetastoreClientDelegate.appendPartition(dbName, tblName, values);
}
 

private void dropSentryPrivileges(
    List<? extends Authorizable> authorizableTable)
    throws SentryUserException, IOException, MetaException {
  String requestorUserName = UserGroupInformation.getCurrentUser()
      .getShortUserName();
  SentryPolicyServiceClient sentryClient = getSentryServiceClient();
  sentryClient.dropPrivileges(requestorUserName, authorizableTable);

  // Close the connection after dropping privileges is done.
  sentryClient.close();
}
 

private void setupMockWarehouseForPath(Path path, boolean isDir, boolean isDefaultDbPath) throws MetaException {
  when(wh.getDnsPath(path)).thenReturn(path);
  when(wh.isDir(path)).thenReturn(isDir);
  when(wh.mkdirs(path, true)).thenReturn(true);
  if (isDefaultDbPath) {
    when(wh.getDefaultDatabasePath(any(String.class))).thenReturn(path);
  }
}
 

@Test
public void databaseWhitelisting() throws Exception {
  String writableDatabase = "writable_db";

  localServer.createDatabase(writableDatabase);

  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_AND_WRITE_ON_DATABASE_WHITELIST,
          writableDatabase)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  Database readOnlyDB = proxy.getDatabase(LOCAL_DATABASE);
  assertNotNull(readOnlyDB);
  Database writableDB = proxy.getDatabase(writableDatabase);
  assertNotNull(writableDB);
  try {
    proxy.alterDatabase(LOCAL_DATABASE, new Database(readOnlyDB));
    fail("Alter DB should not be allowed");
  } catch (MetaException e) {
    // expected
    proxy.reconnect();
  }
  proxy.alterDatabase(writableDatabase, new Database(writableDB));
}
 

private void checkTableStatus(TableDescription tableDescription) throws MetaException {
  String status = tableDescription.getTableStatus();

  if ("CREATING".equals(status) || "DELETING".equals(status)) {
    throw new MetaException("Table " + tableDescription.getTableName() + " is in state "
        + status);
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AggrStats get_aggr_stats_for(PartitionsStatsRequest request)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(request.getDbName());
  return mapping.getClient().get_aggr_stats_for(mapping.transformInboundPartitionsStatsRequest(request));
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Table get_table(String dbname, String tbl_name) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(dbname);
  return mapping
      .transformOutboundTable(mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name));
}
 

@Override
public void dropConstraint(
    String dbName,
    String tblName,
    String constraintName
) throws MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.dropConstraint(dbName, tblName, constraintName);
}
 

@Override
public Map<String, String> partitionNameToSpec(String name) throws MetaException, TException {
    // Lifted from HiveMetaStore
    if (name.length() == 0) {
        return new HashMap<String, String>();
    }
    return Warehouse.makeSpecFromName(name);
}
 

private List<String> getPartitionNames(List<FieldSchema> partitionKeys, List<Partition> partitions)
  throws MetaException {
  List<String> partitionNames = new ArrayList<>(partitions.size());
  for (Partition partition : partitions) {
    partitionNames.add(Warehouse.makePartName(partitionKeys, partition.getValues()));
  }
  return partitionNames;
}
 

@Override
public List<org.apache.hadoop.hive.metastore.api.Table> getTableObjectsByName(String dbName, List<String> tableNames) throws MetaException,
      InvalidOperationException, UnknownDBException, TException {
    List<org.apache.hadoop.hive.metastore.api.Table> hiveTables = Lists.newArrayList();
    for(String tableName : tableNames) {
        hiveTables.add(getTable(dbName, tableName));
    }

    return hiveTables;
}
 

/**
 * Create the column family if it doesn't exist.
 * @param ks
 * @return
 * @throws MetaException
 */
public CfDef createCFIfNotFound(KsDef ks) throws MetaException {
  CfDef cf = getColumnFamily(ks);
  if (cf == null) {
    return createColumnFamily();
  } else {
    return cf;
  }
}
 

@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> dropPartitions(
    String dbName,
    String tblName,
    List<ObjectPair<Integer, byte[]>> partExprs,
    boolean deleteData,
    boolean ifExists,
    boolean needResults
) throws NoSuchObjectException, MetaException, TException {
  return dropPartitions_core(dbName, tblName, partExprs, deleteData, false);
}
 

/**
 * Constructor.
 *
 * @param address                    address
 * @param hiveMetastoreClientFactory hiveMetastoreClientFactory
 * @throws MetaException exception
 */
public MetacatHiveClient(final URI address,
                         final HiveMetastoreClientFactory hiveMetastoreClientFactory)
        throws MetaException {
    this.hiveMetastoreClientFactory = hiveMetastoreClientFactory;
    Preconditions.checkArgument(address.getHost() != null, "metastoreUri host is missing: " + address);
    Preconditions.checkArgument(address.getPort() != -1, "metastoreUri port is missing: " + address);
    this.host = address.getHost();
    this.port = address.getPort();
}
 

@Override
public AWSGlue newClient() throws MetaException {
  try {
    AWSGlueClientBuilder glueClientBuilder = AWSGlueClientBuilder.standard()
        .withCredentials(getAWSCredentialsProvider(conf));

    String regionStr = getProperty(AWS_REGION, conf);
    String glueEndpoint = getProperty(AWS_GLUE_ENDPOINT, conf);

    // ClientBuilder only allows one of EndpointConfiguration or Region to be set
    if (StringUtils.isNotBlank(glueEndpoint)) {
      logger.info("Setting glue service endpoint to " + glueEndpoint);
      glueClientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(glueEndpoint, null));
    } else if (StringUtils.isNotBlank(regionStr)) {
      logger.info("Setting region to : " + regionStr);
      glueClientBuilder.setRegion(regionStr);
    } else {
      Region currentRegion = Regions.getCurrentRegion();
      if (currentRegion != null) {
        logger.info("Using region from ec2 metadata : " + currentRegion.getName());
        glueClientBuilder.setRegion(currentRegion.getName());
      } else {
        logger.info("No region info found, using SDK default region: us-east-1");
      }
    }

    glueClientBuilder.setClientConfiguration(buildClientConfiguration(conf));
    return glueClientBuilder.build();
  } catch (Exception e) {
    String message = "Unable to build AWSGlueClient: " + e;
    logger.error(message);
    throw new MetaException(message);
  }
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(req.getDbName());
  PartitionsByExprResult result = mapping
      .getClient()
      .get_partitions_by_expr(mapping.transformInboundPartitionsByExprRequest(req));
  return mapping.transformOutboundPartitionsByExprResult(result);
}
 

@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void markPartitionForEvent(
    String db_name,
    String tbl_name,
    Map<String, String> part_vals,
    PartitionEventType eventType)
    throws MetaException, NoSuchObjectException, UnknownDBException, UnknownTableException, UnknownPartitionException,
    InvalidPartitionException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  mapping
      .getClient()
      .markPartitionForEvent(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals, eventType);
}