Java Code Examples for org.apache.hadoop.hive.metastore.HiveMetaStoreClient

The following examples show how to use org.apache.hadoop.hive.metastore.HiveMetaStoreClient. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private IMetaStoreClient createMetaStoreClient() throws MetaException {
  HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
    @Override
    public HiveMetaHook getHook(Table tbl) throws MetaException {
      if (tbl == null) {
        return null;
      }

      try {
        HiveStorageHandler storageHandler =
            HiveUtils.getStorageHandler(hiveConf, tbl.getParameters().get(META_TABLE_STORAGE));
        return storageHandler == null ? null : storageHandler.getMetaHook();
      } catch (HiveException e) {
        LOG.error(e.toString());
        throw new MetaException("Failed to get storage handler: " + e);
      }
    }
  };

  return RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
}
 
Example 2
Source Project: iceberg   Source File: HiveClientPool.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected HiveMetaStoreClient newClient()  {
  try {
    return new HiveMetaStoreClient(hiveConf);
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
  } catch (Throwable t) {
    if (t.getMessage().contains("Another instance of Derby may have already booted")) {
      throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " +
          "Derby supports only one client at a time. To fix this, use a metastore that supports " +
          "multiple clients.");
    }

    throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore");
  }
}
 
Example 3
Source Project: griffin   Source File: HiveMetaStoreProxy.java    License: Apache License 2.0 6 votes vote down vote up
@Bean
public IMetaStoreClient initHiveMetastoreClient() {
    HiveConf hiveConf = new HiveConf();
    hiveConf.set("hive.metastore.local", "false");
    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES,
        3);
    hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
    hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts);
    hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval);
    try {
        client = HiveMetaStoreClient.newSynchronizedClient(new HiveMetaStoreClient(hiveConf));
    } catch (Exception e) {
        LOGGER.error("Failed to connect hive metastore. {}", e);
    }
    return client;
}
 
Example 4
Source Project: beeju   Source File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void dropTable() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  createUnpartitionedTable(DATABASE, tableName, server);

  try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
      Statement statement = connection.createStatement()) {
    String dropHql = String.format("DROP TABLE %s.%s", DATABASE, tableName);
    statement.execute(dropHql);
  }

  HiveMetaStoreClient client = server.getCore().newClient();
  try {
    client.getTable(DATABASE, tableName);
    fail(String.format("Table %s.%s was not deleted", DATABASE, tableName));
  } catch (NoSuchObjectException e) {
    // expected
  } finally {
    client.close();
  }
  server.shutdown();
}
 
Example 5
Source Project: beeju   Source File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void dropDatabase() throws Exception {
  HiveServer2Core server = setupServer();
  String databaseName = "Another_DB";

  server.getCore().createDatabase(databaseName);
  try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
      Statement statement = connection.createStatement()) {
    String dropHql = String.format("DROP DATABASE %s", databaseName);
    statement.execute(dropHql);
  }

  HiveMetaStoreClient client = server.getCore().newClient();
  try {
    client.getDatabase(databaseName);
    fail(String.format("Database %s was not deleted", databaseName));
  } catch (NoSuchObjectException e) {
    // expected
  } finally {
    client.close();
  }
  server.shutdown();
}
 
Example 6
Source Project: beeju   Source File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void addPartition() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  createPartitionedTable(DATABASE, tableName, server);

  try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
      Statement statement = connection.createStatement()) {
    String addPartitionHql = String.format("ALTER TABLE %s.%s ADD PARTITION (partcol=1)", DATABASE, tableName);
    statement.execute(addPartitionHql);
  }

  HiveMetaStoreClient client = server.getCore().newClient();
  try {
    List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1);
    assertThat(partitions.size(), is(1));
    assertThat(partitions.get(0).getDbName(), is(DATABASE));
    assertThat(partitions.get(0).getTableName(), is(tableName));
    assertThat(partitions.get(0).getValues(), is(Arrays.asList("1")));
    assertThat(partitions.get(0).getSd().getLocation(),
        is(String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName)));
  } finally {
    client.close();
  }
  server.shutdown();
}
 
Example 7
Source Project: beeju   Source File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
private Table createUnpartitionedTable(String databaseName, String tableName, HiveServer2Core server)
    throws Exception {
  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example 8
Source Project: beeju   Source File: HiveServer2CoreTest.java    License: Apache License 2.0 6 votes vote down vote up
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example 9
@Before
public void init() throws Exception {
  sourceTempUri = temporaryFolder.newFolder("source-temp");
  sourceWarehouseUri = temporaryFolder.newFolder("source-warehouse");
  sourceTableUri = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_TABLE);
  createSourceTable();

  replicaWarehouseUri = temporaryFolder.newFolder("replica-warehouse");
  replicaTableUri = new File(replicaWarehouseUri, DATABASE + "/" + REPLICA_TABLE);
  createReplicaTable();

  ymlFile = temporaryFolder.newFile("diff.yml");

  HiveMetaStoreClient client = catalog.client();
  LOG.info(">>>> Source Table = {}", client.getTable(DATABASE, SOURCE_TABLE));
  LOG.info(">>>> Replica Table = {}", client.getTable(DATABASE, REPLICA_TABLE));

  // Create hive config
  testHiveSiteXml = new HiveXml(catalog.getThriftConnectionUri(), catalog.connectionURL(), catalog.driverClassName())
      .create()
      .getCanonicalPath();

  outputFile = temporaryFolder.newFile("diff.out");
}
 
Example 10
@Before
public void init() throws Exception {
  // We mock the checksum function because LocalFileSystem doesn't compute checksums
  when(checksumFunction.apply(any(Path.class))).thenAnswer(new Answer<String>() {
    @Override
    public String answer(InvocationOnMock invocation) throws Throwable {
      Path path = (Path) invocation.getArguments()[0];
      return path.getName();
    }
  });

  sourceWarehouseUri = temporaryFolder.newFolder("source-warehouse");
  sourceTableUri = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_TABLE);
  createTable(DATABASE, SOURCE_TABLE, sourceTableUri, null, null, false);

  replicaWarehouseUri = temporaryFolder.newFolder("replica-warehouse");
  replicaTableUri = new File(replicaWarehouseUri, DATABASE + "/" + REPLICA_TABLE);
  createTable(DATABASE, REPLICA_TABLE, replicaTableUri, DATABASE + "." + SOURCE_TABLE,
      sourceTableUri.toURI().toString(), true);

  HiveMetaStoreClient client = catalog.client();
  LOG.info(">>>> Source Table = {}", client.getTable(DATABASE, SOURCE_TABLE));
  LOG.info(">>>> Replica Table = {}", client.getTable(DATABASE, REPLICA_TABLE));
}
 
Example 11
Source Project: iceberg   Source File: HiveTableBaseTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException,
        TException,
        InvocationTargetException,
        NoSuchMethodException,
        IllegalAccessException,
        NoSuchFieldException, SQLException {
  this.executorService = Executors.newSingleThreadExecutor();
  hiveLocalDir = createTempDirectory("hive", asFileAttribute(fromString("rwxrwxrwx"))).toFile();
  setupDB("jdbc:derby:" + getDerbyPath() + ";create=true");

  this.server = thriftServer();
  executorService.submit(() -> server.serve());

  this.metastoreClient = new HiveMetaStoreClient(this.hiveConf);
  createIfNotExistsCatalog("hive");
  this.metastoreClient.createDatabase(new Database(DB_NAME, "description", getDBPath(), new HashMap<>()));
  new HiveTables(this.hiveConf).create(schema, partitionSpec, DB_NAME, TABLE_NAME);
}
 
Example 12
public Table makeMetastoreTableObject(HiveMetaStoreClient client,
    String dbName, String tabName, List<FieldSchema> cols) throws Exception {
  Table tbl = new Table();
  tbl.setDbName(dbName);
  tbl.setTableName(tabName);
  StorageDescriptor sd = new StorageDescriptor();
  tbl.setSd(sd);
  tbl.setParameters(new HashMap<String, String>());
  sd.setCols(cols);
  sd.setCompressed(false);
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());
  sd.getSerdeInfo().setName(tbl.getTableName());
  sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  sd.getSerdeInfo().getParameters()
      .put(serdeConstants.SERIALIZATION_FORMAT, "1");
  sd.setSortCols(new ArrayList<Order>());
  return tbl;
}
 
Example 13
Source Project: waggle-dance   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
static Table createPartitionedTable(HiveMetaStoreClient metaStoreClient, String database, String table, File location)
  throws Exception {

  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  hiveTable.setPartitionKeys(PARTITION_COLUMNS);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 
Example 14
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
private void createRemoteTable(File tableUri) throws Exception {
  HiveMetaStoreClient client = remoteServer.client();

  Table hiveTable = createPartitionedTable(client, REMOTE_DATABASE, REMOTE_TABLE, tableUri);

  File partitionEurope = new File(tableUri, "continent=Europe");
  File partitionUk = new File(partitionEurope, "country=UK");

  File partitionAsia = new File(tableUri, "continent=Asia");
  File partitionChina = new File(partitionAsia, "country=China");

  LOG
      .info(">>>> Partitions added: {}",
          client
              .add_partitions(Arrays
                  .asList(newPartition(hiveTable, Arrays.asList("Europe", "UK"), partitionUk),
                      newPartition(hiveTable, Arrays.asList("Asia", "China"), partitionChina))));
}
 
Example 15
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example 16
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void usePrefix() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri())
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = PREFIXED_REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example 17
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void createDatabaseUsingManualAndWhitelistingUpdatesConfig() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 
Example 18
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void createDatabaseDatabaseUsingPrefixAndWhitelistingUpdates() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 
Example 19
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void alterTableOnFederatedIsNotAllowedUsingManual() throws Exception {
  String[] mappableDatabases = new String[] { REMOTE_DATABASE };
  String[] writableDatabaseWhitelist = new String[] { REMOTE_DATABASE };
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.MANUAL)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .federate("doesNotMatterManualMode", remoteServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_ON_DATABASE_WHITELIST, mappableDatabases, writableDatabaseWhitelist)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  Table table = proxy.getTable(REMOTE_DATABASE, REMOTE_TABLE);
  Table newTable = new Table(table);
  newTable.setTableName("new_remote_table");

  proxy.alter_table_with_environmentContext(REMOTE_DATABASE, REMOTE_TABLE, newTable, null);
  assertThat(proxy.tableExists(REMOTE_DATABASE, "new_remote_table"), is(true));
}
 
Example 20
Source Project: waggle-dance   Source File: WaggleDanceIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void getDatabaseFromPatternPrefixed() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .overwriteConfigOnShutdown(false)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), "remote.?database")
      .federate("third", remoteServer.getThriftConnectionUri(), "no_match.*")
      .build();
  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  List<String> allDatabases = proxy.getAllDatabases();

  List<String> expected = Lists.newArrayList("default", LOCAL_DATABASE, PREFIXED_REMOTE_DATABASE);
  assertThat(allDatabases, is(expected));
}
 
Example 21
Source Project: hadoop-etl-udfs   Source File: HiveAdapter.java    License: MIT License 6 votes vote down vote up
static SchemaMetadata readMetadata(SchemaMetadataInfo schemaMetaInfo, ExaConnectionInformation connection, List<String> tableNames, String newSchema) throws SQLException, MetadataException {
    String databaseName = newSchema;
    if (newSchema == null) {
        databaseName = HiveAdapterProperties.getSchema(schemaMetaInfo.getProperties());
    }
    HiveMetaStoreClient hiveMetastoreClient = HiveMetastoreService.getHiveMetastoreClient(connection.getAddress(), HiveQueryGenerator.isKerberosAuth(connection.getPassword()), HiveAdapterProperties.getConnectionName(schemaMetaInfo.getProperties()));
    if (tableNames == null) {
        try {
            tableNames = hiveMetastoreClient.getAllTables(databaseName);
        } catch (MetaException e) {
            e.getMessage();
        }
    }
    List<TableMetadata> tables = new ArrayList<>();
    if (tableNames != null) {
        for (String tableName : tableNames) {
            Table table = HiveTableInformation.getHiveTable(hiveMetastoreClient, tableName, databaseName);
            tables.add(HiveTableInformation.getTableMetadataFromHCatTableMetadata(tableName.toUpperCase(), HiveTableInformation.getHCatTableMetadata(table)));
        }
    }
    return new SchemaMetadata("", tables);
}
 
Example 22
Source Project: streamline   Source File: HiveMetadataService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates secure {@link HiveMetadataService}, which delegates to {@link HiveMetaStoreClient}
 * instantiated with the {@link HiveConf} provided using the first parameter
 */
public static HiveMetadataService newInstance(HiveConf hiveConf, SecurityContext securityContext,
                                              Subject subject, Component hiveMetastore,
                                              Collection<ComponentProcess> hiveMetastoreProcesses)
            throws MetaException, IOException, EntityNotFoundException, PrivilegedActionException {

    if (SecurityUtil.isKerberosAuthenticated(securityContext)) {
        UserGroupInformation.setConfiguration(hiveConf);    // Sets Kerberos rules
        UserGroupInformation.getUGIFromSubject(subject);    // Adds User principal to this subject

        return new HiveMetadataService(
                SecurityUtil.execute(() -> new HiveMetaStoreClient(hiveConf), securityContext, subject),
                    hiveConf, securityContext, subject, hiveMetastore, hiveMetastoreProcesses);
    } else {
        return new HiveMetadataService(new HiveMetaStoreClient(hiveConf), hiveConf, securityContext, subject,
                hiveMetastore, hiveMetastoreProcesses);
    }
}
 
Example 23
Source Project: incubator-sentry   Source File: TestMetaStoreWithPigHCat.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME);
  FileOutputStream to = new FileOutputStream(dataFile);
  Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to);
  to.close();

  policyFile = setAdminOnServer1(ADMINGROUP);
  policyFile
      .addRolesToGroup(USERGROUP1, db_all_role)
      .addRolesToGroup(USERGROUP2, "read_db_role")
      .addPermissionsToRole(db_all_role, "server=server1->db=" + dbName)
      .addPermissionsToRole("read_db_role",
          "server=server1->db=" + dbName + "->table=" + tabName2 + "->action=SELECT")
      .setUserGroupMapping(StaticUserGroup.getStaticMapping());
  writePolicyFile(policyFile);

  HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
  client.dropDatabase(dbName, true, true, true);
  createMetastoreDB(client, dbName);
  client.close();
}
 
Example 24
Source Project: incubator-gobblin   Source File: HivePurgerPublisher.java    License: Apache License 2.0 6 votes vote down vote up
public void initHiveMetastoreClient() throws Exception {
  if (this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION)) {
    String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
    String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM);
    String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
    log.info("Establishing MetastoreClient connection using " + keytabLocation);

    UserGroupInformation.loginUserFromKeytab(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation);
    UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
    loginUser.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws TException {
        HivePurgerPublisher.this.client = new HiveMetaStoreClient(new HiveConf());
        return null;
      }
    });
  } else {
    HivePurgerPublisher.this.client = new HiveMetaStoreClient(new HiveConf());
  }
}
 
Example 25
Source Project: iceberg   Source File: HiveClientPool.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected HiveMetaStoreClient reconnect(HiveMetaStoreClient client) {
  try {
    client.close();
    client.reconnect();
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to reconnect to Hive Metastore");
  }
  return client;
}
 
Example 26
Source Project: iceberg   Source File: HiveMetastoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void startMetastore() throws Exception {
  HiveMetastoreTest.metastore = new TestHiveMetastore();
  metastore.start();
  HiveMetastoreTest.hiveConf = metastore.hiveConf();
  HiveMetastoreTest.metastoreClient = new HiveMetaStoreClient(hiveConf);
  String dbPath = metastore.getDatabasePath(DB_NAME);
  Database db = new Database(DB_NAME, "description", dbPath, new HashMap<>());
  metastoreClient.createDatabase(db);
  HiveMetastoreTest.catalog = new HiveCatalog(hiveConf);
}
 
Example 27
Source Project: pxf   Source File: HiveMetadataFetcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setupCompressionFactory() throws MetaException {

    @SuppressWarnings("unchecked")
    Map<String, String> mockProfileMap = mock(Map.class);
    PluginConf mockPluginConf = mock(PluginConf.class);

    configuration = new Configuration();

    context = new RequestContext();
    context.setPluginConf(mockPluginConf);
    when(mockPluginConf.getPlugins("HiveText")).thenReturn(mockProfileMap);
    when(mockProfileMap.get("OUTPUTFORMAT")).thenReturn("org.greenplum.pxf.api.io.Text");

    mockHiveClient = mock(HiveMetaStoreClient.class);

    context.setConfig("default");
    context.setServerName("default");
    context.setUser("dummy");
    context.setAdditionalConfigProps(null);
    mockConfigurationFactory = mock(ConfigurationFactory.class);

    when(mockConfigurationFactory.initConfiguration("default", "default", "dummy", null))
            .thenReturn(configuration);

    mockClientFactory = mock(HiveClientWrapper.HiveClientFactory.class);

    fakeHiveClientWrapper = new HiveClientWrapper(mockClientFactory);

    when(mockClientFactory.initHiveClient(any())).thenReturn(mockHiveClient);
}
 
Example 28
Source Project: kite   Source File: HiveTestUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void assertTableIsExternal(
    HiveMetaStoreClient client, String db, String name)
    throws MetaException, TException {
  final Table table = client.getTable(db, name);
  Assert.assertTrue("Table should be external db:" + db + " table:" + table,
      MetaStoreUtils.isExternalTable(table) &&
      TableType.EXTERNAL_TABLE.toString().equals(table.getTableType()));
}
 
Example 29
Source Project: beeju   Source File: ThriftHiveMetaStoreCore.java    License: Apache License 2.0 5 votes vote down vote up
public void initialise() throws Exception {
  thriftPort = -1;
  final Lock startLock = new ReentrantLock();
  final Condition startCondition = startLock.newCondition();
  final AtomicBoolean startedServing = new AtomicBoolean();
  try (ServerSocket socket = new ServerSocket(0)) {
    thriftPort = socket.getLocalPort();
  }
  beejuCore.setHiveVar(HiveConf.ConfVars.METASTOREURIS, getThriftConnectionUri());
  final HiveConf hiveConf = new HiveConf(beejuCore.conf(), HiveMetaStoreClient.class);
  thriftServer.execute(new Runnable() {
    @Override
    public void run() {
      try {
        HadoopThriftAuthBridge bridge = new HadoopThriftAuthBridge23();
        HiveMetaStore.startMetaStore(thriftPort, bridge, hiveConf, startLock, startCondition, startedServing);
      } catch (Throwable e) {
        LOG.error("Unable to start a Thrift server for Hive Metastore", e);
      }
    }
  });
  int i = 0;
  while (i++ < 3) {
    startLock.lock();
    try {
      if (startCondition.await(1, TimeUnit.MINUTES)) {
        break;
      }
    } finally {
      startLock.unlock();
    }
    if (i == 3) {
      throw new RuntimeException("Maximum number of tries reached whilst waiting for Thrift server to be ready");
    }
  }
}
 
Example 30
Source Project: beeju   Source File: HiveMetaStoreCore.java    License: Apache License 2.0 5 votes vote down vote up
public void initialise() throws InterruptedException, ExecutionException {
  HiveConf hiveConf = new HiveConf(beejuCore.conf(), HiveMetaStoreClient.class);
  ExecutorService singleThreadExecutor = Executors.newSingleThreadExecutor();
  try {
    client = singleThreadExecutor.submit(new CallableHiveClient(hiveConf)).get();
  } finally {
    singleThreadExecutor.shutdown();
  }
}