Java Code Examples for org.apache.hadoop.hive.metastore.api.MetaException

The following examples show how to use org.apache.hadoop.hive.metastore.api.MetaException. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-gobblin   Source File: HiveProxyQueryExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void setProxiedConnection(final List<String> proxies)
    throws IOException, InterruptedException, TException {
  Preconditions.checkArgument(this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
      "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
  String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM);
  UserGroupInformation loginUser = UserGroupInformation
      .loginUserFromKeytabAndReturnUGI(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation);
  loginUser.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run()
        throws MetaException, SQLException, ClassNotFoundException {
      for (String proxy : proxies) {
        HiveConnection hiveConnection = getHiveConnection(Optional.fromNullable(proxy));
        Statement statement = hiveConnection.createStatement();
        statementMap.put(proxy, statement);
        connectionMap.put(proxy, hiveConnection);
        for (String setting : settings) {
          statement.execute(setting);
        }
      }
      return null;
    }
  });
}
 
Example 2
Source Project: metacat   Source File: HiveConnectorTableService.java    License: Apache License 2.0 6 votes vote down vote up
private HiveStorageFormat extractHiveStorageFormat(final Table table) throws MetaException {
    final StorageDescriptor descriptor = table.getSd();
    if (descriptor == null) {
        throw new MetaException("Table is missing storage descriptor");
    }
    final SerDeInfo serdeInfo = descriptor.getSerdeInfo();
    if (serdeInfo == null) {
        throw new MetaException(
            "Table storage descriptor is missing SerDe info");
    }
    final String outputFormat = descriptor.getOutputFormat();
    final String serializationLib = serdeInfo.getSerializationLib();

    for (HiveStorageFormat format : HiveStorageFormat.values()) {
        if (format.getOutputFormat().equals(outputFormat) && format.getSerde().equals(serializationLib)) {
            return format;
        }
    }
    throw new MetaException(
        String.format("Output format %s with SerDe %s is not supported", outputFormat, serializationLib));
}
 
Example 3
public static String buildExpressionFromPartialSpecification(org.apache.hadoop.hive.metastore.api.Table table,
        List<String> partitionValues) throws MetaException {

  List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys = table.getPartitionKeys();

  if (partitionValues == null || partitionValues.isEmpty() ) {
    return null;
  }

  if (partitionKeys == null || partitionValues.size() > partitionKeys.size()) {
    throw new MetaException("Incorrect number of partition values: " + partitionValues);
  }

  partitionKeys = partitionKeys.subList(0, partitionValues.size());
  List<String> predicates = new LinkedList<>();
  for (int i = 0; i < partitionValues.size(); i++) {
    if (!Strings.isNullOrEmpty(partitionValues.get(i))) {
      predicates.add(buildPredicate(partitionKeys.get(i), partitionValues.get(i)));
    }
  }

  return JOINER.join(predicates);
}
 
Example 4
private void performDropPartitionPostProcessing(String dbName, String tblName,
                                                org.apache.hadoop.hive.metastore.api.Partition partition, boolean deleteData, boolean ifPurge)
      throws MetaException, NoSuchObjectException, TException {
    if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
        Path partPath = new Path(partition.getSd().getLocation());
        org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
        if (isExternalTable(table)){
            //Don't delete external table data
            return;
        }
        boolean mustPurge = isMustPurge(table, ifPurge);
        wh.deleteDir(partPath, true, mustPurge);
        try {
            List<String> values = partition.getValues();
            deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
        } catch (IOException e) {
            throw new MetaException(e.getMessage());
        }
    }
}
 
Example 5
@Override
public void onCreateDatabase(CreateDatabaseEvent dbEvent)
    throws MetaException {

  // don't sync paths/privileges if the operation has failed
  if (!dbEvent.getStatus()) {
    LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
      " since the operation failed. \n");
    return;
  }

  if (dbEvent.getDatabase().getLocationUri() != null) {
    String authzObj = dbEvent.getDatabase().getName();
    String path = dbEvent.getDatabase().getLocationUri();
    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
      plugin.addPath(authzObj, path);
    }
  }
  // drop the privileges on the database, in case anything left behind during
  // last drop db
  if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
    return;
  }

  dropSentryDbPrivileges(dbEvent.getDatabase().getName());
}
 
Example 6
/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
 
Example 7
private void performDropPartitionPostProcessing(
    String dbName,
    String tblName,
    org.apache.hadoop.hive.metastore.api.Partition partition,
    boolean deleteData,
    boolean ifPurge
) throws TException {
  if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
    Path partPath = new Path(partition.getSd().getLocation());
    org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
    if (isExternalTable(table)) {
      //Don't delete external table data
      return;
    }
    boolean mustPurge = isMustPurge(table, ifPurge);
    wh.deleteDir(partPath, true, mustPurge);
    try {
      List<String> values = partition.getValues();
      deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
    } catch (IOException e) {
      throw new MetaException(e.getMessage());
    }
  }
}
 
Example 8
@Test
public void testCheckTableSchemaNullSerializationValid() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
      "col2:dynamo_col2#,hashKey:hashKey");
  parameters.put(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, "true");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "array<bigint>", ""));
  cols.add(new FieldSchema("hashKey", "string", ""));
  sd.setCols(cols);
  table.setSd(sd);

  // This check is expected to pass for the given input
  storageHandler.checkTableSchemaType(description, table);
}
 
Example 9
@Override
public void preCreateTable(Table table) throws MetaException {
  DynamoDBClient client = createDynamoDBClient(table);
  try {

    boolean isExternal = MetaStoreUtils.isExternalTable(table);

    if (!isExternal) {
      throw new MetaException("Only EXTERNAL tables are supported for DynamoDB.");
    }

    String tableName = HiveDynamoDBUtil.getDynamoDBTableName(table.getParameters()
        .get(DynamoDBConstants.TABLE_NAME), table.getTableName());
    TableDescription tableDescription = client.describeTable(tableName);

    checkTableStatus(tableDescription);
    checkTableSchemaMapping(tableDescription, table);
    checkTableSchemaType(tableDescription, table);
  } finally {
    client.close();
  }
}
 
Example 10
Source Project: iceberg   Source File: HiveClientPool.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected HiveMetaStoreClient newClient()  {
  try {
    return new HiveMetaStoreClient(hiveConf);
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
  } catch (Throwable t) {
    if (t.getMessage().contains("Another instance of Derby may have already booted")) {
      throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " +
          "Derby supports only one client at a time. To fix this, use a metastore that supports " +
          "multiple clients.");
    }

    throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore");
  }
}
 
Example 11
@Override
public TProcessor getProcessor(TTransport transport) {
  try {
    if (transport instanceof TSocket) {
      Socket socket = ((TSocket) transport).getSocket();
      log.debug("Received a connection from ip: {}", socket.getInetAddress().getHostAddress());
    }
    CloseableIHMSHandler baseHandler = federatedHMSHandlerFactory.create();
    IHMSHandler handler = newRetryingHMSHandler(ExceptionWrappingHMSHandler.newProxyInstance(baseHandler), hiveConf,
        false);
    transportMonitor.monitor(transport, baseHandler);
    return new TSetIpAddressProcessor<>(handler);
  } catch (MetaException | ReflectiveOperationException | RuntimeException e) {
    throw new RuntimeException("Error creating TProcessor", e);
  }
}
 
Example 12
@Override
public org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet get_privilege_set(
    HiveObjectRef obj,
    String user, List<String> groups
) throws MetaException, TException {
  return glueMetastoreClientDelegate.getPrivilegeSet(obj, user, groups);
}
 
Example 13
Source Project: dremio-oss   Source File: HiveClientWithAuthz.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void connect() throws MetaException {
  doAsCommand(
    (PrivilegedExceptionAction<Void>) () -> {
      client = Hive.get(hiveConf).getMSC();
      return null;
    },
      ugiForRpc,
      "Failed to connect to Hive metastore"
  );

  // Hive authorization helper needs the query user
  // (not the user from metastore client used to communicate with metastore)
  this.authorizer = new HiveAuthorizationHelper(client, hiveConf, userName);
}
 
Example 14
@Override
public org.apache.hadoop.hive.metastore.api.Partition appendPartition(
    String dbName,
    String tblName,
    List<String> values
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException, MetaException, TException {
  return glueMetastoreClientDelegate.appendPartition(dbName, tblName, values);
}
 
Example 15
private void dropSentryPrivileges(
    List<? extends Authorizable> authorizableTable)
    throws SentryUserException, IOException, MetaException {
  String requestorUserName = UserGroupInformation.getCurrentUser()
      .getShortUserName();
  SentryPolicyServiceClient sentryClient = getSentryServiceClient();
  sentryClient.dropPrivileges(requestorUserName, authorizableTable);

  // Close the connection after dropping privileges is done.
  sentryClient.close();
}
 
Example 16
private void setupMockWarehouseForPath(Path path, boolean isDir, boolean isDefaultDbPath) throws MetaException {
  when(wh.getDnsPath(path)).thenReturn(path);
  when(wh.isDir(path)).thenReturn(isDir);
  when(wh.mkdirs(path, true)).thenReturn(true);
  if (isDefaultDbPath) {
    when(wh.getDefaultDatabasePath(any(String.class))).thenReturn(path);
  }
}
 
Example 17
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void databaseWhitelisting() throws Exception {
  String writableDatabase = "writable_db";

  localServer.createDatabase(writableDatabase);

  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_AND_WRITE_ON_DATABASE_WHITELIST,
          writableDatabase)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  Database readOnlyDB = proxy.getDatabase(LOCAL_DATABASE);
  assertNotNull(readOnlyDB);
  Database writableDB = proxy.getDatabase(writableDatabase);
  assertNotNull(writableDB);
  try {
    proxy.alterDatabase(LOCAL_DATABASE, new Database(readOnlyDB));
    fail("Alter DB should not be allowed");
  } catch (MetaException e) {
    // expected
    proxy.reconnect();
  }
  proxy.alterDatabase(writableDatabase, new Database(writableDB));
}
 
Example 18
private void checkTableStatus(TableDescription tableDescription) throws MetaException {
  String status = tableDescription.getTableStatus();

  if ("CREATING".equals(status) || "DELETING".equals(status)) {
    throw new MetaException("Table " + tableDescription.getTableName() + " is in state "
        + status);
  }
}
 
Example 19
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AggrStats get_aggr_stats_for(PartitionsStatsRequest request)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(request.getDbName());
  return mapping.getClient().get_aggr_stats_for(mapping.transformInboundPartitionsStatsRequest(request));
}
 
Example 20
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Table get_table(String dbname, String tbl_name) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(dbname);
  return mapping
      .transformOutboundTable(mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name));
}
 
Example 21
@Override
public void dropConstraint(
    String dbName,
    String tblName,
    String constraintName
) throws MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.dropConstraint(dbName, tblName, constraintName);
}
 
Example 22
@Override
public Map<String, String> partitionNameToSpec(String name) throws MetaException, TException {
    // Lifted from HiveMetaStore
    if (name.length() == 0) {
        return new HashMap<String, String>();
    }
    return Warehouse.makeSpecFromName(name);
}
 
Example 23
Source Project: circus-train   Source File: HiveEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
private List<String> getPartitionNames(List<FieldSchema> partitionKeys, List<Partition> partitions)
  throws MetaException {
  List<String> partitionNames = new ArrayList<>(partitions.size());
  for (Partition partition : partitions) {
    partitionNames.add(Warehouse.makePartName(partitionKeys, partition.getValues()));
  }
  return partitionNames;
}
 
Example 24
@Override
public List<org.apache.hadoop.hive.metastore.api.Table> getTableObjectsByName(String dbName, List<String> tableNames) throws MetaException,
      InvalidOperationException, UnknownDBException, TException {
    List<org.apache.hadoop.hive.metastore.api.Table> hiveTables = Lists.newArrayList();
    for(String tableName : tableNames) {
        hiveTables.add(getTable(dbName, tableName));
    }

    return hiveTables;
}
 
Example 25
Source Project: Hive-Cassandra   Source File: CassandraManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create the column family if it doesn't exist.
 * @param ks
 * @return
 * @throws MetaException
 */
public CfDef createCFIfNotFound(KsDef ks) throws MetaException {
  CfDef cf = getColumnFamily(ks);
  if (cf == null) {
    return createColumnFamily();
  } else {
    return cf;
  }
}
 
Example 26
@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> dropPartitions(
    String dbName,
    String tblName,
    List<ObjectPair<Integer, byte[]>> partExprs,
    boolean deleteData,
    boolean ifExists,
    boolean needResults
) throws NoSuchObjectException, MetaException, TException {
  return dropPartitions_core(dbName, tblName, partExprs, deleteData, false);
}
 
Example 27
Source Project: metacat   Source File: MetacatHiveClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor.
 *
 * @param address                    address
 * @param hiveMetastoreClientFactory hiveMetastoreClientFactory
 * @throws MetaException exception
 */
public MetacatHiveClient(final URI address,
                         final HiveMetastoreClientFactory hiveMetastoreClientFactory)
        throws MetaException {
    this.hiveMetastoreClientFactory = hiveMetastoreClientFactory;
    Preconditions.checkArgument(address.getHost() != null, "metastoreUri host is missing: " + address);
    Preconditions.checkArgument(address.getPort() != -1, "metastoreUri port is missing: " + address);
    this.host = address.getHost();
    this.port = address.getPort();
}
 
Example 28
@Override
public AWSGlue newClient() throws MetaException {
  try {
    AWSGlueClientBuilder glueClientBuilder = AWSGlueClientBuilder.standard()
        .withCredentials(getAWSCredentialsProvider(conf));

    String regionStr = getProperty(AWS_REGION, conf);
    String glueEndpoint = getProperty(AWS_GLUE_ENDPOINT, conf);

    // ClientBuilder only allows one of EndpointConfiguration or Region to be set
    if (StringUtils.isNotBlank(glueEndpoint)) {
      logger.info("Setting glue service endpoint to " + glueEndpoint);
      glueClientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(glueEndpoint, null));
    } else if (StringUtils.isNotBlank(regionStr)) {
      logger.info("Setting region to : " + regionStr);
      glueClientBuilder.setRegion(regionStr);
    } else {
      Region currentRegion = Regions.getCurrentRegion();
      if (currentRegion != null) {
        logger.info("Using region from ec2 metadata : " + currentRegion.getName());
        glueClientBuilder.setRegion(currentRegion.getName());
      } else {
        logger.info("No region info found, using SDK default region: us-east-1");
      }
    }

    glueClientBuilder.setClientConfiguration(buildClientConfiguration(conf));
    return glueClientBuilder.build();
  } catch (Exception e) {
    String message = "Unable to build AWSGlueClient: " + e;
    logger.error(message);
    throw new MetaException(message);
  }
}
 
Example 29
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(req.getDbName());
  PartitionsByExprResult result = mapping
      .getClient()
      .get_partitions_by_expr(mapping.transformInboundPartitionsByExprRequest(req));
  return mapping.transformOutboundPartitionsByExprResult(result);
}
 
Example 30
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void markPartitionForEvent(
    String db_name,
    String tbl_name,
    Map<String, String> part_vals,
    PartitionEventType eventType)
    throws MetaException, NoSuchObjectException, UnknownDBException, UnknownTableException, UnknownPartitionException,
    InvalidPartitionException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  mapping
      .getClient()
      .markPartitionForEvent(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals, eventType);
}