org.apache.hadoop.hive.metastore.api.MetaException Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.MetaException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: incubator-gobblin   Author: apache   File: HiveProxyQueryExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void setProxiedConnection(final List<String> proxies)
    throws IOException, InterruptedException, TException {
  Preconditions.checkArgument(this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION),
      "Missing required property " + ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
  String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
  String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM);
  UserGroupInformation loginUser = UserGroupInformation
      .loginUserFromKeytabAndReturnUGI(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation);
  loginUser.doAs(new PrivilegedExceptionAction<Void>() {
    @Override
    public Void run()
        throws MetaException, SQLException, ClassNotFoundException {
      for (String proxy : proxies) {
        HiveConnection hiveConnection = getHiveConnection(Optional.fromNullable(proxy));
        Statement statement = hiveConnection.createStatement();
        statementMap.put(proxy, statement);
        connectionMap.put(proxy, hiveConnection);
        for (String setting : settings) {
          statement.execute(setting);
        }
      }
      return null;
    }
  });
}
 
Example #2
Source Project: metacat   Author: Netflix   File: HiveConnectorTableService.java    License: Apache License 2.0 6 votes vote down vote up
private HiveStorageFormat extractHiveStorageFormat(final Table table) throws MetaException {
    final StorageDescriptor descriptor = table.getSd();
    if (descriptor == null) {
        throw new MetaException("Table is missing storage descriptor");
    }
    final SerDeInfo serdeInfo = descriptor.getSerdeInfo();
    if (serdeInfo == null) {
        throw new MetaException(
            "Table storage descriptor is missing SerDe info");
    }
    final String outputFormat = descriptor.getOutputFormat();
    final String serializationLib = serdeInfo.getSerializationLib();

    for (HiveStorageFormat format : HiveStorageFormat.values()) {
        if (format.getOutputFormat().equals(outputFormat) && format.getSerde().equals(serializationLib)) {
            return format;
        }
    }
    throw new MetaException(
        String.format("Output format %s with SerDe %s is not supported", outputFormat, serializationLib));
}
 
Example #3
public static String buildExpressionFromPartialSpecification(org.apache.hadoop.hive.metastore.api.Table table,
        List<String> partitionValues) throws MetaException {

  List<org.apache.hadoop.hive.metastore.api.FieldSchema> partitionKeys = table.getPartitionKeys();

  if (partitionValues == null || partitionValues.isEmpty() ) {
    return null;
  }

  if (partitionKeys == null || partitionValues.size() > partitionKeys.size()) {
    throw new MetaException("Incorrect number of partition values: " + partitionValues);
  }

  partitionKeys = partitionKeys.subList(0, partitionValues.size());
  List<String> predicates = new LinkedList<>();
  for (int i = 0; i < partitionValues.size(); i++) {
    if (!Strings.isNullOrEmpty(partitionValues.get(i))) {
      predicates.add(buildPredicate(partitionKeys.get(i), partitionValues.get(i)));
    }
  }

  return JOINER.join(predicates);
}
 
Example #4
private void performDropPartitionPostProcessing(String dbName, String tblName,
                                                org.apache.hadoop.hive.metastore.api.Partition partition, boolean deleteData, boolean ifPurge)
      throws MetaException, NoSuchObjectException, TException {
    if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
        Path partPath = new Path(partition.getSd().getLocation());
        org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
        if (isExternalTable(table)){
            //Don't delete external table data
            return;
        }
        boolean mustPurge = isMustPurge(table, ifPurge);
        wh.deleteDir(partPath, true, mustPurge);
        try {
            List<String> values = partition.getValues();
            deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
        } catch (IOException e) {
            throw new MetaException(e.getMessage());
        }
    }
}
 
Example #5
Source Project: incubator-sentry   Author: apache   File: SentryMetastorePostEventListener.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void onCreateDatabase(CreateDatabaseEvent dbEvent)
    throws MetaException {

  // don't sync paths/privileges if the operation has failed
  if (!dbEvent.getStatus()) {
    LOGGER.debug("Skip syncing paths/privileges with Sentry server for onCreateDatabase event," +
      " since the operation failed. \n");
    return;
  }

  if (dbEvent.getDatabase().getLocationUri() != null) {
    String authzObj = dbEvent.getDatabase().getName();
    String path = dbEvent.getDatabase().getLocationUri();
    for (SentryMetastoreListenerPlugin plugin : sentryPlugins) {
      plugin.addPath(authzObj, path);
    }
  }
  // drop the privileges on the database, in case anything left behind during
  // last drop db
  if (!syncWithPolicyStore(AuthzConfVars.AUTHZ_SYNC_CREATE_WITH_POLICY_STORE)) {
    return;
  }

  dropSentryDbPrivileges(dbEvent.getDatabase().getName());
}
 
Example #6
/**
 * Taken from HiveMetaStore#append_partition_common
 */
private org.apache.hadoop.hive.metastore.api.Partition buildPartitionFromValues(
  org.apache.hadoop.hive.metastore.api.Table table, List<String> values) throws MetaException {
  org.apache.hadoop.hive.metastore.api.Partition partition = new org.apache.hadoop.hive.metastore.api.Partition();
  partition.setDbName(table.getDbName());
  partition.setTableName(table.getTableName());
  partition.setValues(values);
  partition.setSd(table.getSd().deepCopy());

  Path partLocation = new Path(table.getSd().getLocation(), Warehouse.makePartName(table.getPartitionKeys(), values));
  partition.getSd().setLocation(partLocation.toString());

  long timeInSecond = System.currentTimeMillis() / MILLISECOND_TO_SECOND_FACTOR;
  partition.setCreateTime((int) timeInSecond);
  partition.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(timeInSecond));
  return partition;
}
 
Example #7
private void performDropPartitionPostProcessing(
    String dbName,
    String tblName,
    org.apache.hadoop.hive.metastore.api.Partition partition,
    boolean deleteData,
    boolean ifPurge
) throws TException {
  if (deleteData && partition.getSd() != null && partition.getSd().getLocation() != null) {
    Path partPath = new Path(partition.getSd().getLocation());
    org.apache.hadoop.hive.metastore.api.Table table = getTable(dbName, tblName);
    if (isExternalTable(table)) {
      //Don't delete external table data
      return;
    }
    boolean mustPurge = isMustPurge(table, ifPurge);
    wh.deleteDir(partPath, true, mustPurge);
    try {
      List<String> values = partition.getValues();
      deleteParentRecursive(partPath.getParent(), values.size() - 1, mustPurge);
    } catch (IOException e) {
      throw new MetaException(e.getMessage());
    }
  }
}
 
Example #8
Source Project: emr-dynamodb-connector   Author: awslabs   File: DynamoDBStorageHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCheckTableSchemaNullSerializationValid() throws MetaException {
  TableDescription description = getHashRangeTable();

  Table table = new Table();
  Map<String, String> parameters = Maps.newHashMap();
  parameters.put(DynamoDBConstants.DYNAMODB_COLUMN_MAPPING, "col1:dynamo_col1$," +
      "col2:dynamo_col2#,hashKey:hashKey");
  parameters.put(DynamoDBConstants.DYNAMODB_NULL_SERIALIZATION, "true");
  table.setParameters(parameters);
  StorageDescriptor sd = new StorageDescriptor();
  List<FieldSchema> cols = Lists.newArrayList();
  cols.add(new FieldSchema("col1", "string", ""));
  cols.add(new FieldSchema("col2", "array<bigint>", ""));
  cols.add(new FieldSchema("hashKey", "string", ""));
  sd.setCols(cols);
  table.setSd(sd);

  // This check is expected to pass for the given input
  storageHandler.checkTableSchemaType(description, table);
}
 
Example #9
Source Project: emr-dynamodb-connector   Author: awslabs   File: DynamoDBStorageHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void preCreateTable(Table table) throws MetaException {
  DynamoDBClient client = createDynamoDBClient(table);
  try {

    boolean isExternal = MetaStoreUtils.isExternalTable(table);

    if (!isExternal) {
      throw new MetaException("Only EXTERNAL tables are supported for DynamoDB.");
    }

    String tableName = HiveDynamoDBUtil.getDynamoDBTableName(table.getParameters()
        .get(DynamoDBConstants.TABLE_NAME), table.getTableName());
    TableDescription tableDescription = client.describeTable(tableName);

    checkTableStatus(tableDescription);
    checkTableSchemaMapping(tableDescription, table);
    checkTableSchemaType(tableDescription, table);
  } finally {
    client.close();
  }
}
 
Example #10
Source Project: iceberg   Author: apache   File: HiveClientPool.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected HiveMetaStoreClient newClient()  {
  try {
    return new HiveMetaStoreClient(hiveConf);
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
  } catch (Throwable t) {
    if (t.getMessage().contains("Another instance of Derby may have already booted")) {
      throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " +
          "Derby supports only one client at a time. To fix this, use a metastore that supports " +
          "multiple clients.");
    }

    throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore");
  }
}
 
Example #11
Source Project: waggle-dance   Author: HotelsDotCom   File: TSetIpAddressProcessorFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public TProcessor getProcessor(TTransport transport) {
  try {
    if (transport instanceof TSocket) {
      Socket socket = ((TSocket) transport).getSocket();
      log.debug("Received a connection from ip: {}", socket.getInetAddress().getHostAddress());
    }
    CloseableIHMSHandler baseHandler = federatedHMSHandlerFactory.create();
    IHMSHandler handler = newRetryingHMSHandler(ExceptionWrappingHMSHandler.newProxyInstance(baseHandler), hiveConf,
        false);
    transportMonitor.monitor(transport, baseHandler);
    return new TSetIpAddressProcessor<>(handler);
  } catch (MetaException | ReflectiveOperationException | RuntimeException e) {
    throw new RuntimeException("Error creating TProcessor", e);
  }
}
 
Example #12
@Override
public org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet get_privilege_set(
    HiveObjectRef obj,
    String user, List<String> groups
) throws MetaException, TException {
  return glueMetastoreClientDelegate.getPrivilegeSet(obj, user, groups);
}
 
Example #13
Source Project: dremio-oss   Author: dremio   File: HiveClientWithAuthz.java    License: Apache License 2.0 5 votes vote down vote up
@Override
void connect() throws MetaException {
  doAsCommand(
    (PrivilegedExceptionAction<Void>) () -> {
      client = Hive.get(hiveConf).getMSC();
      return null;
    },
      ugiForRpc,
      "Failed to connect to Hive metastore"
  );

  // Hive authorization helper needs the query user
  // (not the user from metastore client used to communicate with metastore)
  this.authorizer = new HiveAuthorizationHelper(client, hiveConf, userName);
}
 
Example #14
@Override
public org.apache.hadoop.hive.metastore.api.Partition appendPartition(
    String dbName,
    String tblName,
    List<String> values
) throws InvalidObjectException, org.apache.hadoop.hive.metastore.api.AlreadyExistsException, MetaException, TException {
  return glueMetastoreClientDelegate.appendPartition(dbName, tblName, values);
}
 
Example #15
Source Project: incubator-sentry   Author: apache   File: SentryMetastorePostEventListener.java    License: Apache License 2.0 5 votes vote down vote up
private void dropSentryPrivileges(
    List<? extends Authorizable> authorizableTable)
    throws SentryUserException, IOException, MetaException {
  String requestorUserName = UserGroupInformation.getCurrentUser()
      .getShortUserName();
  SentryPolicyServiceClient sentryClient = getSentryServiceClient();
  sentryClient.dropPrivileges(requestorUserName, authorizableTable);

  // Close the connection after dropping privileges is done.
  sentryClient.close();
}
 
Example #16
private void setupMockWarehouseForPath(Path path, boolean isDir, boolean isDefaultDbPath) throws MetaException {
  when(wh.getDnsPath(path)).thenReturn(path);
  when(wh.isDir(path)).thenReturn(isDir);
  when(wh.mkdirs(path, true)).thenReturn(true);
  if (isDefaultDbPath) {
    when(wh.getDefaultDatabasePath(any(String.class))).thenReturn(path);
  }
}
 
Example #17
Source Project: waggle-dance   Author: HotelsDotCom   File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void databaseWhitelisting() throws Exception {
  String writableDatabase = "writable_db";

  localServer.createDatabase(writableDatabase);

  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_AND_WRITE_ON_DATABASE_WHITELIST,
          writableDatabase)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  Database readOnlyDB = proxy.getDatabase(LOCAL_DATABASE);
  assertNotNull(readOnlyDB);
  Database writableDB = proxy.getDatabase(writableDatabase);
  assertNotNull(writableDB);
  try {
    proxy.alterDatabase(LOCAL_DATABASE, new Database(readOnlyDB));
    fail("Alter DB should not be allowed");
  } catch (MetaException e) {
    // expected
    proxy.reconnect();
  }
  proxy.alterDatabase(writableDatabase, new Database(writableDB));
}
 
Example #18
Source Project: emr-dynamodb-connector   Author: awslabs   File: DynamoDBStorageHandler.java    License: Apache License 2.0 5 votes vote down vote up
private void checkTableStatus(TableDescription tableDescription) throws MetaException {
  String status = tableDescription.getTableStatus();

  if ("CREATING".equals(status) || "DELETING".equals(status)) {
    throw new MetaException("Table " + tableDescription.getTableName() + " is in state "
        + status);
  }
}
 
Example #19
Source Project: waggle-dance   Author: HotelsDotCom   File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AggrStats get_aggr_stats_for(PartitionsStatsRequest request)
    throws NoSuchObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(request.getDbName());
  return mapping.getClient().get_aggr_stats_for(mapping.transformInboundPartitionsStatsRequest(request));
}
 
Example #20
Source Project: waggle-dance   Author: HotelsDotCom   File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Table get_table(String dbname, String tbl_name) throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(dbname);
  return mapping
      .transformOutboundTable(mapping.getClient().get_table(mapping.transformInboundDatabaseName(dbname), tbl_name));
}
 
Example #21
@Override
public void dropConstraint(
    String dbName,
    String tblName,
    String constraintName
) throws MetaException, NoSuchObjectException, TException {
  glueMetastoreClientDelegate.dropConstraint(dbName, tblName, constraintName);
}
 
Example #22
@Override
public Map<String, String> partitionNameToSpec(String name) throws MetaException, TException {
    // Lifted from HiveMetaStore
    if (name.length() == 0) {
        return new HashMap<String, String>();
    }
    return Warehouse.makeSpecFromName(name);
}
 
Example #23
Source Project: circus-train   Author: HotelsDotCom   File: HiveEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
private List<String> getPartitionNames(List<FieldSchema> partitionKeys, List<Partition> partitions)
  throws MetaException {
  List<String> partitionNames = new ArrayList<>(partitions.size());
  for (Partition partition : partitions) {
    partitionNames.add(Warehouse.makePartName(partitionKeys, partition.getValues()));
  }
  return partitionNames;
}
 
Example #24
@Override
public List<org.apache.hadoop.hive.metastore.api.Table> getTableObjectsByName(String dbName, List<String> tableNames) throws MetaException,
      InvalidOperationException, UnknownDBException, TException {
    List<org.apache.hadoop.hive.metastore.api.Table> hiveTables = Lists.newArrayList();
    for(String tableName : tableNames) {
        hiveTables.add(getTable(dbName, tableName));
    }

    return hiveTables;
}
 
Example #25
Source Project: Hive-Cassandra   Author: dvasilen   File: CassandraManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create the column family if it doesn't exist.
 * @param ks
 * @return
 * @throws MetaException
 */
public CfDef createCFIfNotFound(KsDef ks) throws MetaException {
  CfDef cf = getColumnFamily(ks);
  if (cf == null) {
    return createColumnFamily();
  } else {
    return cf;
  }
}
 
Example #26
@Override
public List<org.apache.hadoop.hive.metastore.api.Partition> dropPartitions(
    String dbName,
    String tblName,
    List<ObjectPair<Integer, byte[]>> partExprs,
    boolean deleteData,
    boolean ifExists,
    boolean needResults
) throws NoSuchObjectException, MetaException, TException {
  return dropPartitions_core(dbName, tblName, partExprs, deleteData, false);
}
 
Example #27
Source Project: metacat   Author: Netflix   File: MetacatHiveClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor.
 *
 * @param address                    address
 * @param hiveMetastoreClientFactory hiveMetastoreClientFactory
 * @throws MetaException exception
 */
public MetacatHiveClient(final URI address,
                         final HiveMetastoreClientFactory hiveMetastoreClientFactory)
        throws MetaException {
    this.hiveMetastoreClientFactory = hiveMetastoreClientFactory;
    Preconditions.checkArgument(address.getHost() != null, "metastoreUri host is missing: " + address);
    Preconditions.checkArgument(address.getPort() != -1, "metastoreUri port is missing: " + address);
    this.host = address.getHost();
    this.port = address.getPort();
}
 
Example #28
@Override
public AWSGlue newClient() throws MetaException {
  try {
    AWSGlueClientBuilder glueClientBuilder = AWSGlueClientBuilder.standard()
        .withCredentials(getAWSCredentialsProvider(conf));

    String regionStr = getProperty(AWS_REGION, conf);
    String glueEndpoint = getProperty(AWS_GLUE_ENDPOINT, conf);

    // ClientBuilder only allows one of EndpointConfiguration or Region to be set
    if (StringUtils.isNotBlank(glueEndpoint)) {
      logger.info("Setting glue service endpoint to " + glueEndpoint);
      glueClientBuilder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(glueEndpoint, null));
    } else if (StringUtils.isNotBlank(regionStr)) {
      logger.info("Setting region to : " + regionStr);
      glueClientBuilder.setRegion(regionStr);
    } else {
      Region currentRegion = Regions.getCurrentRegion();
      if (currentRegion != null) {
        logger.info("Using region from ec2 metadata : " + currentRegion.getName());
        glueClientBuilder.setRegion(currentRegion.getName());
      } else {
        logger.info("No region info found, using SDK default region: us-east-1");
      }
    }

    glueClientBuilder.setClientConfiguration(buildClientConfiguration(conf));
    return glueClientBuilder.build();
  } catch (Exception e) {
    String message = "Unable to build AWSGlueClient: " + e;
    logger.error(message);
    throw new MetaException(message);
  }
}
 
Example #29
Source Project: waggle-dance   Author: HotelsDotCom   File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req)
    throws MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = databaseMappingService.databaseMapping(req.getDbName());
  PartitionsByExprResult result = mapping
      .getClient()
      .get_partitions_by_expr(mapping.transformInboundPartitionsByExprRequest(req));
  return mapping.transformOutboundPartitionsByExprResult(result);
}
 
Example #30
Source Project: waggle-dance   Author: HotelsDotCom   File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void markPartitionForEvent(
    String db_name,
    String tbl_name,
    Map<String, String> part_vals,
    PartitionEventType eventType)
    throws MetaException, NoSuchObjectException, UnknownDBException, UnknownTableException, UnknownPartitionException,
    InvalidPartitionException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  mapping
      .getClient()
      .markPartitionForEvent(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals, eventType);
}