Java Code Examples for org.apache.hadoop.hive.metastore.api.Database

The following examples show how to use org.apache.hadoop.hive.metastore.api.Database. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-atlas   Source File: HiveMetaStoreBridge.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks if db is already registered, else creates and registers db entity
 * @param databaseName
 * @return
 * @throws Exception
 */
private Referenceable registerDatabase(String databaseName) throws Exception {
    Referenceable dbRef = getDatabaseReference(clusterName, databaseName);
    Database db = hiveClient.getDatabase(databaseName);

    if (db != null) {
        if (dbRef == null) {
            dbRef = createDBInstance(db);
            dbRef = registerInstance(dbRef);
        } else {
            LOG.info("Database {} is already registered with id {}. Updating it.", databaseName, dbRef.getId().id);
            dbRef = createOrUpdateDBInstance(db, dbRef);
            updateInstance(dbRef);
        }
    }
    return dbRef;
}
 
Example 2
@Test
public void testCreate() throws TException {
  HiveConf hiveConf = new HiveConf();
  HiveMetaStoreClientFactory factory = new HiveMetaStoreClientFactory(hiveConf);

  // Since we havE a specified hive-site in the classpath, so have to null it out here to proceed the test
  // The original value it will get if no local hive-site is placed, will be an empty string. 
  hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "");
  hiveConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, "");
  IMetaStoreClient msc = factory.create();

  String dbName = "test_db";
  String description = "test database";
  String location = "file:/tmp/" + dbName;
  Database db = new Database(dbName, description, location, null);

  msc.dropDatabase(dbName, true, true);
  msc.createDatabase(db);
  db = msc.getDatabase(dbName);
  Assert.assertEquals(db.getName(), dbName);
  Assert.assertEquals(db.getDescription(), description);
  Assert.assertEquals(db.getLocationUri(), location);
}
 
Example 3
Source Project: presto   Source File: InMemoryThriftMetastore.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void alterDatabase(HiveIdentity identity, String databaseName, Database newDatabase)
{
    String newDatabaseName = newDatabase.getName();

    if (databaseName.equals(newDatabaseName)) {
        if (databases.replace(databaseName, newDatabase) == null) {
            throw new SchemaNotFoundException(databaseName);
        }
        return;
    }

    Database database = databases.get(databaseName);
    if (database == null) {
        throw new SchemaNotFoundException(databaseName);
    }
    if (databases.putIfAbsent(newDatabaseName, database) != null) {
        throw new SchemaAlreadyExistsException(newDatabaseName);
    }
    databases.remove(databaseName);

    rewriteKeys(relations, name -> new SchemaTableName(newDatabaseName, name.getTableName()));
    rewriteKeys(views, name -> new SchemaTableName(newDatabaseName, name.getTableName()));
    rewriteKeys(partitions, name -> name.withSchemaName(newDatabaseName));
    rewriteKeys(tablePrivileges, name -> name.withDatabase(newDatabaseName));
}
 
Example 4
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void alterDatabase(String databaseName, CatalogDatabase newDatabase, boolean ignoreIfNotExists)
		throws DatabaseNotExistException, CatalogException {
	checkArgument(!StringUtils.isNullOrWhitespaceOnly(databaseName), "databaseName cannot be null or empty");
	checkNotNull(newDatabase, "newDatabase cannot be null");

	// client.alterDatabase doesn't throw any exception if there is no existing database
	if (!databaseExists(databaseName)) {
		if (!ignoreIfNotExists) {
			throw new DatabaseNotExistException(getName(), databaseName);
		}

		return;
	}

	Database newHiveDatabase = instantiateHiveDatabase(databaseName, newDatabase);

	try {
		client.alterDatabase(databaseName, newHiveDatabase);
	} catch (TException e) {
		throw new CatalogException(String.format("Failed to alter database %s", databaseName), e);
	}
}
 
Example 5
Source Project: iceberg   Source File: TestHiveCatalog.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSetNamespaceProperties() throws TException {
  Namespace namespace = Namespace.of("dbname_set");

  catalog.createNamespace(namespace, meta);
  catalog.setProperties(namespace,
      ImmutableMap.of(
          "owner", "alter_apache",
          "test", "test",
          "location", "file:/data/tmp",
          "comment", "iceberg test")
  );

  Database database = metastoreClient.getDatabase(namespace.level(0));
  Assert.assertEquals(database.getParameters().get("owner"), "alter_apache");
  Assert.assertEquals(database.getParameters().get("test"), "test");
  Assert.assertEquals(database.getParameters().get("group"), "iceberg");
  AssertHelpers.assertThrows("Should fail to namespace not exist" + namespace,
      NoSuchNamespaceException.class, "Namespace does not exist: ", () -> {
        catalog.setProperties(Namespace.of("db2", "db2", "ns2"), meta);
      });
}
 
Example 6
Source Project: iceberg   Source File: TestIcebergSourceHiveTables.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void startMetastoreAndSpark() throws Exception {
  TestIcebergSourceHiveTables.metastore = new TestHiveMetastore();
  metastore.start();
  TestIcebergSourceHiveTables.hiveConf = metastore.hiveConf();
  String dbPath = metastore.getDatabasePath("db");
  Database db = new Database("db", "desc", dbPath, new HashMap<>());
  TestIcebergSourceHiveTables.clients = new HiveClientPool(1, hiveConf);
  clients.run(client -> {
    client.createDatabase(db);
    return null;
  });

  TestIcebergSourceHiveTables.spark = SparkSession.builder()
      .master("local[2]")
      .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname))
      .getOrCreate();

  TestIcebergSourceHiveTables.catalog = new HiveCatalog(hiveConf);
}
 
Example 7
@Override
public void doTask() throws Exception {
  Database db = hmsHandler.get_database(dbName);
  List<String> dbPath = PathsUpdate.parsePath(db.getLocationUri());
  if (dbPath != null) {
    synchronized (update) {
      Preconditions.checkArgument(dbName.equalsIgnoreCase(db.getName()));
      update.newPathChange(dbName).addToAddPaths(dbPath);
    }
  }
  List<String> allTblStr = hmsHandler.get_all_tables(dbName);
  for (int i = 0; i < allTblStr.size(); i += maxTablesPerCall) {
    List<String> tablesToFetch =
            allTblStr.subList(i, Math.min(
                    i + maxTablesPerCall, allTblStr.size()));
    Callable<CallResult> tableTask =
            new TableTask(db, tablesToFetch, update);
    synchronized (results) {
      results.add(threadPool.submit(tableTask));
    }
  }
}
 
Example 8
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void createDatabaseUsingManualAndWhitelistingUpdatesConfig() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 
Example 9
Source Project: atlas   Source File: HiveMetaStoreBridgeTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testImportThatUpdatesRegisteredDatabase() throws Exception {
    // setup database
    when(hiveClient.getAllDatabases()).thenReturn(Arrays.asList(new String[]{TEST_DB_NAME}));
    String description = "This is a default database";
    Database db = new Database(TEST_DB_NAME, description, "/user/hive/default", null);
    when(hiveClient.getDatabase(TEST_DB_NAME)).thenReturn(db);
    when(hiveClient.getAllTables(TEST_DB_NAME)).thenReturn(Arrays.asList(new String[]{}));

    returnExistingDatabase(TEST_DB_NAME, atlasClientV2, METADATA_NAMESPACE);

    when(atlasEntityWithExtInfo.getEntity("72e06b34-9151-4023-aa9d-b82103a50e76"))
            .thenReturn((new AtlasEntity.AtlasEntityWithExtInfo(
                    getEntity(HiveDataTypes.HIVE_DB.getName(), AtlasClient.GUID, "72e06b34-9151-4023-aa9d-b82103a50e76"))).getEntity());

    HiveMetaStoreBridge bridge = new HiveMetaStoreBridge(METADATA_NAMESPACE, hiveClient, atlasClientV2);
    bridge.importHiveMetadata(null, null, true);

    // verify update is called
    verify(atlasClientV2).updateEntity(anyObject());
}
 
Example 10
Source Project: iceberg   Source File: HiveTableBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException,
        TException,
        InvocationTargetException,
        NoSuchMethodException,
        IllegalAccessException,
        NoSuchFieldException, SQLException {
  this.executorService = Executors.newSingleThreadExecutor();
  hiveLocalDir = createTempDirectory("hive", asFileAttribute(fromString("rwxrwxrwx"))).toFile();
  setupDB("jdbc:derby:" + getDerbyPath() + ";create=true");

  this.server = thriftServer();
  executorService.submit(() -> server.serve());

  this.metastoreClient = new HiveMetaStoreClient(this.hiveConf);
  createIfNotExistsCatalog("hive");
  this.metastoreClient.createDatabase(new Database(DB_NAME, "description", getDBPath(), new HashMap<>()));
  new HiveTables(this.hiveConf).create(schema, partitionSpec, DB_NAME, TABLE_NAME);
}
 
Example 11
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void createDatabaseDatabaseUsingPrefixAndWhitelistingUpdates() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 
Example 12
Source Project: flink   Source File: HiveDialectITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateDatabase() throws Exception {
	tableEnv.executeSql("create database db1 comment 'db1 comment'");
	Database db = hiveCatalog.getHiveDatabase("db1");
	assertEquals("db1 comment", db.getDescription());
	assertFalse(Boolean.parseBoolean(db.getParameters().get(CatalogConfig.IS_GENERIC)));

	String db2Location = warehouse + "/db2_location";
	tableEnv.executeSql(String.format("create database db2 location '%s' with dbproperties('k1'='v1')", db2Location));
	db = hiveCatalog.getHiveDatabase("db2");
	assertEquals(db2Location, locationPath(db.getLocationUri()));
	assertEquals("v1", db.getParameters().get("k1"));
}
 
Example 13
public static com.amazonaws.services.glue.model.Database convertDatabase(Database hiveDatabase) {
  com.amazonaws.services.glue.model.Database catalogDatabase = new com.amazonaws.services.glue.model.Database();
  catalogDatabase.setName(hiveDatabase.getName());
  catalogDatabase.setDescription(hiveDatabase.getDescription());
  catalogDatabase.setLocationUri(hiveDatabase.getLocationUri());
  catalogDatabase.setParameters(hiveDatabase.getParameters());
  return catalogDatabase;
}
 
Example 14
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void primaryMappedDatabasesPrefixed() throws Exception {
  localServer.createDatabase("random_primary");
  remoteServer.createDatabase("random_federated");

  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .withPrimaryMappedDatabases(new String[] { LOCAL_DATABASE })
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  List<String> allDatabases = proxy.getAllDatabases();
  assertThat(allDatabases.size(), is(2));
  assertThat(allDatabases.get(0), is(LOCAL_DATABASE));
  assertThat(allDatabases.get(1), is(PREFIXED_REMOTE_DATABASE));

  // Ensure that the database is added to mapped-databases
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore primaryMetaStore = federations.getPrimaryMetaStore();
  assertThat(primaryMetaStore.getMappedDatabases().contains("newDB"), is(true));
}
 
Example 15
public static DatabaseInput convertToDatabaseInput(com.amazonaws.services.glue.model.Database database) {
  DatabaseInput input = new DatabaseInput();

  input.setName(database.getName());
  input.setDescription(database.getDescription());
  input.setLocationUri(database.getLocationUri());
  input.setParameters(database.getParameters());

  return input;
}
 
Example 16
@BeforeClass
public static void setup() throws MetaException {
  conf = mock(HiveConf.class);
  wh = mock(Warehouse.class);
  tmpPath = new Path("/db");
  when(wh.getDefaultDatabasePath(anyString())).thenReturn(tmpPath);
  when(wh.getDnsPath(any(Path.class))).thenReturn(tmpPath);
  when(wh.isDir(any(Path.class))).thenReturn(true);
  when(wh.getDatabasePath(any(Database.class))).thenReturn(tmpPath);
  when(conf.get(HiveConf.ConfVars.USERS_IN_ADMIN_ROLE.varname,"")).thenReturn("");

  glueClient = new GlueTestClientFactory().newClient();
  GlueClientFactory clientFactory = mock(GlueClientFactory.class);
  when(clientFactory.newClient()).thenReturn(glueClient);

  metastoreClient = new AWSCatalogMetastoreClient.Builder().withHiveConf(conf).withWarehouse(wh)
          .withClientFactory(clientFactory).build();
  catalogDB = getTestDatabase();
  catalogTable = getTestTable();
  hiveDB = CatalogToHiveConverter.convertDatabase(catalogDB);
  hiveTable = CatalogToHiveConverter.convertTable(catalogTable, catalogDB.getName());

  glueClient.createDatabase(new CreateDatabaseRequest()
    .withDatabaseInput(GlueInputConverter.convertToDatabaseInput(catalogDB)));
  glueClient.createTable(new CreateTableRequest()
    .withDatabaseName(catalogDB.getName())
    .withTableInput(GlueInputConverter.convertToTableInput(catalogTable)));
}
 
Example 17
@Test
public void listValidDatabases() throws TException {
  Database database2 = CatalogToHiveConverter.convertDatabase(getTestDatabase());
  additionalDbForCleanup.add(database2.getName());
  metastoreClient.createDatabase(hiveDB);
  metastoreClient.createDatabase(database2);
  List<String> databaseName = metastoreClient.getAllDatabases();
  assertTrue(databaseName.contains(hiveDB.getName()));
  assertTrue(databaseName.contains(database2.getName()));
}
 
Example 18
Source Project: waggle-dance   Source File: FederatedHMSHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
@Loggable(value = Loggable.DEBUG, skipResult = true, name = INVOCATION_LOG_NAME)
public void create_database(Database database)
    throws AlreadyExistsException, InvalidObjectException, MetaException, TException {
  DatabaseMapping mapping = databaseMappingService.primaryDatabaseMapping();
  mapping.createDatabase(mapping.transformInboundDatabase(database));
}
 
Example 19
@Override
public boolean updateTableStatsFast(
    Database db,
    Table tbl,
    Warehouse wh,
    boolean madeDir,
    boolean forceRecompute,
    EnvironmentContext environmentContext
) throws MetaException {
  return MetaStoreUtils.updateTableStatsFast(db, tbl, wh, madeDir, forceRecompute, environmentContext);
}
 
Example 20
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CatalogDatabase getDatabase(String databaseName) throws DatabaseNotExistException, CatalogException {
	Database hiveDatabase = getHiveDatabase(databaseName);

	Map<String, String> properties = hiveDatabase.getParameters();

	boolean isGeneric = isGenericForGet(properties);
	if (!isGeneric) {
		properties.put(SqlCreateHiveDatabase.DATABASE_LOCATION_URI, hiveDatabase.getLocationUri());
	}

	return new CatalogDatabaseImpl(properties, hiveDatabase.getDescription());
}
 
Example 21
Source Project: waggle-dance   Source File: FederatedHMSHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void alter_database() throws TException {
  Database database = new Database();
  database.setName(DB_P);
  Database inboundDB = new Database();
  inboundDB.setName("inbound");
  when(primaryMapping.transformInboundDatabase(database)).thenReturn(inboundDB);
  when(primaryMapping.transformInboundDatabaseName(DB_P)).thenReturn("inbound");

  handler.alter_database(DB_P, database);
  verify(primaryMapping, times(2)).checkWritePermissions(DB_P);
  verify(primaryClient).alter_database("inbound", inboundDB);
}
 
Example 22
Source Project: presto   Source File: MockThriftMetastoreClient.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Database getDatabase(String name)
        throws TException
{
    accessCount.incrementAndGet();
    if (throwException) {
        throw new RuntimeException();
    }
    if (!name.equals(TEST_DATABASE)) {
        throw new NoSuchObjectException();
    }
    return new Database(TEST_DATABASE, null, null, null);
}
 
Example 23
Source Project: flink   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
private static Database instantiateHiveDatabase(String databaseName, CatalogDatabase database) {

		Map<String, String> properties = database.getProperties();

		String dbLocationUri = properties.remove(HiveCatalogConfig.DATABASE_LOCATION_URI);

		return new Database(
			databaseName,
			database.getComment(),
			dbLocationUri,
			properties
		);
	}
 
Example 24
Source Project: iceberg   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean setProperties(Namespace namespace,  Map<String, String> properties) {
  Map<String, String> parameter = Maps.newHashMap();

  parameter.putAll(loadNamespaceMetadata(namespace));
  parameter.putAll(properties);
  Database database = convertToDatabase(namespace, parameter);

  return alterHiveDataBase(namespace, database);
}
 
Example 25
Source Project: iceberg   Source File: HiveCatalog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean removeProperties(Namespace namespace,  Set<String> properties) {
  Map<String, String> parameter = Maps.newHashMap();

  parameter.putAll(loadNamespaceMetadata(namespace));
  properties.forEach(key -> parameter.put(key, null));
  Database database = convertToDatabase(namespace, parameter);

  return alterHiveDataBase(namespace, database);
}
 
Example 26
Source Project: metacat   Source File: HiveConnectorInfoConverter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Converts from DatabaseDto to the connector database.
 *
 * @param databaseInfo Metacat database Info
 * @return connector database
 */
@Override
public Database fromDatabaseInfo(final DatabaseInfo databaseInfo) {
    final QualifiedName databaseName = databaseInfo.getName();
    final String name = (databaseName == null) ? "" : databaseName.getDatabaseName();
    //this is a temp hack to resolve the uri = null issue
    // final String dbUri = Strings.isNullOrEmpty(databaseInfo.getUri()) ? "file://temp/" : databaseInfo.getUri();
    final Map<String, String> metadata
        = (databaseInfo.getMetadata() != null) ? databaseInfo.getMetadata() : Collections.EMPTY_MAP;
    return new Database(name, name, databaseInfo.getUri(), metadata);
}
 
Example 27
Source Project: metacat   Source File: CatalogThriftHiveMetastore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public Database get_database(final String name) throws TException {
    return requestWrapper("get_database", new Object[]{name}, () -> {
        final String databaseName = normalizeIdentifier(name);
        final DatabaseDto dto = v1.getDatabase(catalogName, databaseName, true, false);
        return hiveConverters.metacatToHiveDatabase(dto);
    });
}
 
Example 28
Source Project: iceberg   Source File: HiveMetastoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void startMetastore() throws Exception {
  HiveMetastoreTest.metastore = new TestHiveMetastore();
  metastore.start();
  HiveMetastoreTest.hiveConf = metastore.hiveConf();
  HiveMetastoreTest.metastoreClient = new HiveMetaStoreClient(hiveConf);
  String dbPath = metastore.getDatabasePath(DB_NAME);
  Database db = new Database(DB_NAME, "description", dbPath, new HashMap<>());
  metastoreClient.createDatabase(db);
  HiveMetastoreTest.catalog = new HiveCatalog(hiveConf);
}
 
Example 29
Source Project: metacat   Source File: MetacatHiveClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}.
 */
@Override
public void createDatabase(final Database database) throws TException {
    try (HiveMetastoreClient client = createMetastoreClient()) {
        client.create_database(database);
    }
}
 
Example 30
Source Project: metacat   Source File: CatalogThriftHiveMetastore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public void create_database(final Database database) throws TException {
    requestWrapper("create_database", new Object[]{database}, () -> {
        final String dbName = normalizeIdentifier(database.getName());
        v1.createDatabase(catalogName, dbName,
            DatabaseCreateRequestDto.builder().metadata(database.getParameters()).uri(database.getLocationUri())
                .build());
        return null;
    });
}