org.apache.hadoop.hive.metastore.api.AlreadyExistsException Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.api.AlreadyExistsException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchCreatePartitionsHelper.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
private void processResult(List<PartitionError> partitionErrors) {
  if (partitionErrors == null || partitionErrors.isEmpty()) {
    return;
  }

  logger.error(String.format("BatchCreatePartitions failed to create %d out of %d partitions. \n",
      partitionErrors.size(), partitionMap.size()));

  for (PartitionError partitionError : partitionErrors) {
    Partition partitionFailed = partitionMap.remove(new PartitionKey(partitionError.getPartitionValues()));

    TException exception = CatalogToHiveConverter.errorDetailToHiveException(partitionError.getErrorDetail());
    if (ifNotExists && exception instanceof AlreadyExistsException) {
      // AlreadyExistsException is allowed, so we shouldn't add the partition to partitionsFailed list
      continue;
    }
    logger.error(exception);
    if (firstTException == null) {
      firstTException = exception;
    }
    partitionsFailed.add(partitionFailed);
  }
}
 
Example #2
Source File: GlueMetastoreClientDelegate.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
/**
 * @return boolean
 *    true  -> directory created
 *    false -> directory not created
 */
public boolean validateNewTableAndCreateDirectory(org.apache.hadoop.hive.metastore.api.Table tbl) throws TException {
  checkNotNull(tbl, "tbl cannot be null");
  if (tableExists(tbl.getDbName(), tbl.getTableName())) {
    throw new AlreadyExistsException("Table " + tbl.getTableName() + " already exists.");
  }
  validateTableObject(tbl, conf);

  if (TableType.VIRTUAL_VIEW.toString().equals(tbl.getTableType())) {
    // we don't need to create directory for virtual views
    return false;
  }

  if (StringUtils.isEmpty(tbl.getSd().getLocation())) {
    org.apache.hadoop.hive.metastore.api.Database db = getDatabase(tbl.getDbName());
    tbl.getSd().setLocation(hiveShims.getDefaultTablePath(db, tbl.getTableName(), wh).toString());
  } else {
    tbl.getSd().setLocation(wh.getDnsPath(new Path(tbl.getSd().getLocation())).toString());
  }

  Path tblPath = new Path(tbl.getSd().getLocation());
  return makeDirs(wh, tblPath);
}
 
Example #3
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table(Table tbl)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table(mapping.transformInboundTable(tbl));
}
 
Example #4
Source File: HiveMetaStoreBasedRegister.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private void onPartitionExistWithoutComputingDiff(Table table, Partition nativePartition, TException e) throws TException {
  if(e == null) {
    return;
  }
  if (e instanceof AlreadyExistsException) {
    log.debug(String.format("Partition %s in table %s with location %s already exists and no need to update",
        stringifyPartition(nativePartition), table.getTableName(), nativePartition.getSd().getLocation()));
  }
  else {
    throw e;
  }
}
 
Example #5
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_environment_context(Table tbl, EnvironmentContext environment_context)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_environment_context(mapping.transformInboundTable(tbl), environment_context);
}
 
Example #6
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition(Partition new_part)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping.getClient().add_partition(mapping.transformInboundPartition(new_part));
  return mapping.transformOutboundPartition(result);
}
 
Example #7
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition add_partition_with_environment_context(Partition new_part, EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_part.getDbName());
  Partition result = mapping
      .getClient()
      .add_partition_with_environment_context(mapping.transformInboundPartition(new_part), environment_context);
  return mapping.transformOutboundPartition(result);
}
 
Example #8
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions(List<Partition> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (Partition partition : new_parts) {
      mapping.checkWritePermissions(partition.getDbName());
    }
    return mapping.getClient().add_partitions(mapping.transformInboundPartitions(new_parts));
  }
  return 0;
}
 
Example #9
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public int add_partitions_pspec(List<PartitionSpec> new_parts)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  if (!new_parts.isEmpty()) {
    // Need to pick one mapping and use that for permissions and getting the client.
    // If the partitions added are for different databases in different clients that won't work with waggle-dance
    DatabaseMapping mapping = databaseMappingService.databaseMapping(new_parts.get(0).getDbName());
    for (PartitionSpec partitionSpec : new_parts) {
      mapping.checkWritePermissions(partitionSpec.getDbName());
    }
    return mapping.getClient().add_partitions_pspec(mapping.transformInboundPartitionSpecs(new_parts));
  }
  return 0;
}
 
Example #10
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition(String db_name, String tbl_name, List<String> part_vals)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition result = mapping
      .getClient()
      .append_partition(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals);
  return mapping.transformOutboundPartition(result);
}
 
Example #11
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public AddPartitionsResult add_partitions_req(AddPartitionsRequest request)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(request.getDbName());
  for (Partition partition : request.getParts()) {
    mapping.checkWritePermissions(partition.getDbName());
  }
  AddPartitionsResult result = mapping
      .getClient()
      .add_partitions_req(mapping.transformInboundAddPartitionsRequest(request));
  return mapping.transformOutboundAddPartitionsResult(result);
}
 
Example #12
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_with_environment_context(
    String db_name,
    String tbl_name,
    List<String> part_vals,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name, part_vals,
          environment_context);
  return mapping.transformOutboundPartition(partition);
}
 
Example #13
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name(String db_name, String tbl_name, String part_name)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name(mapping.transformInboundDatabaseName(db_name), tbl_name, part_name);
  return mapping.transformOutboundPartition(partition);
}
 
Example #14
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Partition append_partition_by_name_with_environment_context(
    String db_name,
    String tbl_name,
    String part_name,
    EnvironmentContext environment_context)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(db_name);
  Partition partition = mapping
      .getClient()
      .append_partition_by_name_with_environment_context(mapping.transformInboundDatabaseName(db_name), tbl_name,
          part_name, environment_context);
  return mapping.transformOutboundPartition(partition);
}
 
Example #15
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public Index add_index(Index new_index, Table index_table)
    throws InvalidObjectException, AlreadyExistsException, MetaException, TException {
  DatabaseMapping mapping = checkWritePermissions(new_index.getDbName());
  mapping.checkWritePermissions(index_table.getDbName());
  Index result = mapping
      .getClient()
      .add_index(mapping.transformInboundIndex(new_index), mapping.transformInboundTable(index_table));
  return mapping.transformOutboundIndex(result);
}
 
Example #16
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_function(Function func)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(func.getDbName());
  mapping.getClient().create_function(mapping.transformInboundFunction(func));
}
 
Example #17
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_table_with_constraints(Table tbl, List<SQLPrimaryKey> primaryKeys, List<SQLForeignKey> foreignKeys)
    throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException {
  DatabaseMapping mapping = checkWritePermissions(tbl.getDbName());
  mapping.getClient().create_table_with_constraints(mapping.transformInboundTable(tbl), primaryKeys, foreignKeys);
}
 
Example #18
Source File: MetacatHMSHandler.java    From metacat with Apache License 2.0 5 votes vote down vote up
/**
 * Adds and drops partitions in one transaction.
 *
 * @param databaseName database name
 * @param tableName    table name
 * @param addParts     list of partitions
 * @param dropParts    list of partition values
 * @param deleteData   if true, deletes the data
 * @return true if successful
 * @throws NoSuchObjectException Exception if table does not exists
 * @throws MetaException         Exception if
 * @throws TException            any internal exception
 */
@SuppressWarnings({"checkstyle:methodname"})
public boolean add_drop_partitions(final String databaseName,
                                   final String tableName, final List<Partition> addParts,
                                   final List<List<String>> dropParts, final boolean deleteData)
        throws NoSuchObjectException, MetaException, TException {
    startFunction("add_drop_partitions : db=" + databaseName + " tbl=" + tableName);
    if (addParts.size() == 0 && dropParts.size() == 0) {
        return true;
    }
    for (List<String> partVals : dropParts) {
        LOG.info("Drop Partition values:" + partVals);
    }
    for (Partition part : addParts) {
        LOG.info("Add Partition values:" + part);
    }

    boolean ret = false;
    Exception ex = null;
    try {
        ret = addDropPartitionsCore(getMS(), databaseName, tableName, addParts, dropParts, false, null);
    } catch (Exception e) {
        ex = e;
        if (e instanceof MetaException) {
            throw (MetaException) e;
        } else if (e instanceof InvalidObjectException) {
            throw (InvalidObjectException) e;
        } else if (e instanceof AlreadyExistsException) {
            throw (AlreadyExistsException) e;
        } else if (e instanceof NoSuchObjectException) {
            throw (NoSuchObjectException) e;
        } else {
            throw newMetaException(e);
        }
    } finally {
        endFunction("drop_partitions", ret, ex, tableName);
    }
    return ret;

}
 
Example #19
Source File: MetacatHMSHandler.java    From metacat with Apache License 2.0 5 votes vote down vote up
private boolean startAddPartition(
        final RawStore ms, final Partition part, final boolean ifNotExists) throws MetaException, TException {
    MetaStoreUtils.validatePartitionNameCharacters(part.getValues(),
            partitionValidationPattern);
    final boolean doesExist = ms.doesPartitionExist(
            part.getDbName(), part.getTableName(), part.getValues());
    if (doesExist && !ifNotExists) {
        throw new AlreadyExistsException("Partition already exists: " + part);
    }
    return !doesExist;
}
 
Example #20
Source File: MetacatHMSHandler.java    From metacat with Apache License 2.0 5 votes vote down vote up
private List<Partition> addPartitionsCoreNoTxn(
        final RawStore ms, final Table tbl, final List<Partition> parts, final boolean ifNotExists,
        final Map<PartValEqWrapper, Boolean> addedPartitions, final List<Partition> existingParts)
        throws MetaException, InvalidObjectException, AlreadyExistsException, TException {
    logInfo("add_partitions");
    final String dbName = tbl.getDbName();
    final String tblName = tbl.getTableName();
    final List<Partition> result = new ArrayList<Partition>();
    for (Partition part : parts) {
        if (!part.getTableName().equals(tblName) || !part.getDbName().equals(dbName)) {
            throw new MetaException("Partition does not belong to target table "
                    + dbName + "." + tblName + ": " + part);
        }
        final boolean shouldAdd = startAddPartition(ms, part, ifNotExists);
        if (!shouldAdd) {
            existingParts.add(part);
            LOG.info("Not adding partition " + part + " as it already exists");
            continue;
        }
        final boolean madeDir = createLocationForAddedPartition(tbl, part);
        if (addedPartitions.put(new PartValEqWrapper(part), madeDir) != null) {
            // Technically, for ifNotExists case, we could insert one and discard the other
            // because the first one now "exists", but it seems better to report the problem
            // upstream as such a command doesn't make sense.
            throw new MetaException("Duplicate partitions in the list: " + part);
        }
        initializeAddedPartition(tbl, part, madeDir);
        result.add(part);
    }
    return result;
}
 
Example #21
Source File: TestHiveCleanService.java    From Hue-Ctrip-DI with MIT License 5 votes vote down vote up
private void add_partition(HiveMetaStoreClient client, Table table,
	      List<String> vals, String location) throws InvalidObjectException,
	        AlreadyExistsException, MetaException, TException {

    Partition part = new Partition();
    part.setDbName(table.getDbName());
    part.setTableName(table.getTableName());
    part.setValues(vals);
    part.setParameters(new HashMap<String, String>());
    part.setSd(table.getSd());
    part.getSd().setSerdeInfo(table.getSd().getSerdeInfo());
    part.getSd().setLocation(table.getSd().getLocation() + location);

    client.add_partition(part);
}
 
Example #22
Source File: LocalHiveMetastoreTestUtils.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
public void createTestDb(String dbName) throws Exception {
  Database db = new Database(dbName, "Some description", "/tmp/" + dbName, new HashMap<String, String>());
  try {
    this.localMetastoreClient.createDatabase(db);
  } catch (AlreadyExistsException e) {
    log.warn(dbName + " already exits");
  }
}
 
Example #23
Source File: HivePartitionManagerTest.java    From data-highway with Apache License 2.0 5 votes vote down vote up
@Test
public void addPartition_shouldIgnoreAlreadyExistsException() throws Exception {
  doThrow(AlreadyExistsException.class).when(metaStoreClient).add_partition(any());
  doReturn(URI.create("resolved/location")).when(locationResolver).resolveLocation(LOCATION, false);
  doReturn(Instant.ofEpochSecond(1526462225L)).when(clock).instant();

  Optional<Partition> result = underTest.addPartition(TABLE, PARTITION_VALUES, LOCATION);

  assertThat(result.isPresent(), is(false));
}
 
Example #24
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_database(Database database)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.primaryDatabaseMapping();
  mapping.createDatabase(mapping.transformInboundDatabase(database));
}
 
Example #25
Source File: MetaStoreMappingImpl.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Override
public void createDatabase(Database database)
  throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  if (accessControlHandler.hasCreatePermission()) {
    getClient().create_database(database);
    accessControlHandler.databaseCreatedNotification(database.getName());
  } else {
    throw new NotAllowedException("You cannot create the database '" + database.getName() + "'.");
  }
}
 
Example #26
Source File: GlueMetastoreClientDelegate.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
public void createTableWithConstraints(
    org.apache.hadoop.hive.metastore.api.Table table,
    List<SQLPrimaryKey> primaryKeys,
    List<SQLForeignKey> foreignKeys
) throws AlreadyExistsException, TException {
  throw new UnsupportedOperationException("createTableWithConstraints is not supported");
}
 
Example #27
Source File: CircusTrainReplicationModeIntegrationTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private void addPartitionsToSource(Table sourceTable, URI sourceTableLocation)
  throws IOException, InvalidObjectException, AlreadyExistsException, MetaException, TException {

  final File dataFileUk = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileUk, "1\tadam\tlondon\n2\tsusan\tglasgow\n");
  String partitionEurope = "/continent=Europe";
  String partitionUk = partitionEurope + "/country=UK";
  String partitionFileLocationUk = SOURCE_MANAGED_PARTITIONED_TABLE + partitionUk;
  String fileKeyUk = String.format("%s/%s/%s", DATABASE, partitionFileLocationUk, PART_00000);
  s3Client.putObject("source", fileKeyUk, dataFileUk);

  final File dataFileChina = temporaryFolder.newFile();
  FileUtils.writeStringToFile(dataFileChina, "1\tchun\tbeijing\n2\tshanghai\tmilan\n");
  String partitionAsia = "/continent=Asia";
  String partitionChina = partitionAsia + "/country=China";
  String partitionFileLocation = SOURCE_MANAGED_PARTITIONED_TABLE + partitionChina;
  String fileKeyChina = String.format("%s/%s/%s", DATABASE, partitionFileLocation, PART_00000);
  s3Client.putObject("source", fileKeyChina, dataFileChina);

  URI ukPartitionURI = URI.create(sourceTableLocation + partitionUk);
  URI chinaPartitionURI = URI.create(sourceTableLocation + partitionChina);
  sourceCatalog
      .client()
      .add_partitions(Arrays
          .asList(newTablePartition(sourceTable, Arrays.asList("Europe", "UK"), ukPartitionURI),
              newTablePartition(sourceTable, Arrays.asList("Asia", "China"), chinaPartitionURI)));
}
 
Example #28
Source File: MetastoreClientIndexIntegrationTest.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
@Test (expected = AlreadyExistsException.class)
public void createDuplicateIndex() throws TException {
  assertFalse(indexExist(hiveDB.getName(), hiveTable.getTableName(), hiveIndex.getIndexName()));
  metastoreClient.createIndex(hiveIndex, hiveIndexTable);
  assertTrue(indexExist(hiveDB.getName(), hiveTable.getTableName(), hiveIndex.getIndexName()));

  Table hiveIndexTable2 = CatalogToHiveConverter.convertTable(getTestTable(), hiveDB.getName());
  hiveIndex.setIndexTableName(hiveIndexTable2.getTableName());

  metastoreClient.createIndex(hiveIndex, hiveIndexTable2);
}
 
Example #29
Source File: CatalogToHiveConverter.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 4 votes vote down vote up
public TException get(String msg) {
  return new AlreadyExistsException(msg);
}
 
Example #30
Source File: FederatedHMSHandler.java    From waggle-dance with Apache License 2.0 4 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public boolean create_type(Type type)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  return getPrimaryClient().create_type(type);
}