org.apache.hadoop.hive.metastore.HiveMetaStoreClient Java Examples

The following examples show how to use org.apache.hadoop.hive.metastore.HiveMetaStoreClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HivePurgerPublisher.java    From incubator-gobblin with Apache License 2.0 7 votes vote down vote up
public void initHiveMetastoreClient() throws Exception {
  if (this.state.contains(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION)) {
    String superUser = this.state.getProp(ComplianceConfigurationKeys.GOBBLIN_COMPLIANCE_SUPER_USER);
    String realm = this.state.getProp(ConfigurationKeys.KERBEROS_REALM);
    String keytabLocation = this.state.getProp(ConfigurationKeys.SUPER_USER_KEY_TAB_LOCATION);
    log.info("Establishing MetastoreClient connection using " + keytabLocation);

    UserGroupInformation.loginUserFromKeytab(HostUtils.getPrincipalUsingHostname(superUser, realm), keytabLocation);
    UserGroupInformation loginUser = UserGroupInformation.getLoginUser();
    loginUser.doAs(new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws TException {
        HivePurgerPublisher.this.client = new HiveMetaStoreClient(new HiveConf());
        return null;
      }
    });
  } else {
    HivePurgerPublisher.this.client = new HiveMetaStoreClient(new HiveConf());
  }
}
 
Example #2
Source File: HiveMetaStoreClientFactory.java    From incubator-gobblin with Apache License 2.0 7 votes vote down vote up
private IMetaStoreClient createMetaStoreClient() throws MetaException {
  HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
    @Override
    public HiveMetaHook getHook(Table tbl) throws MetaException {
      if (tbl == null) {
        return null;
      }

      try {
        HiveStorageHandler storageHandler =
            HiveUtils.getStorageHandler(hiveConf, tbl.getParameters().get(META_TABLE_STORAGE));
        return storageHandler == null ? null : storageHandler.getMetaHook();
      } catch (HiveException e) {
        LOG.error(e.toString());
        throw new MetaException("Failed to get storage handler: " + e);
      }
    }
  };

  return RetryingMetaStoreClient.getProxy(hiveConf, hookLoader, HiveMetaStoreClient.class.getName());
}
 
Example #3
Source File: HiveMetadataService.java    From streamline with Apache License 2.0 6 votes vote down vote up
/**
 * Creates secure {@link HiveMetadataService}, which delegates to {@link HiveMetaStoreClient}
 * instantiated with the {@link HiveConf} provided using the first parameter
 */
public static HiveMetadataService newInstance(HiveConf hiveConf, SecurityContext securityContext,
                                              Subject subject, Component hiveMetastore,
                                              Collection<ComponentProcess> hiveMetastoreProcesses)
            throws MetaException, IOException, EntityNotFoundException, PrivilegedActionException {

    if (SecurityUtil.isKerberosAuthenticated(securityContext)) {
        UserGroupInformation.setConfiguration(hiveConf);    // Sets Kerberos rules
        UserGroupInformation.getUGIFromSubject(subject);    // Adds User principal to this subject

        return new HiveMetadataService(
                SecurityUtil.execute(() -> new HiveMetaStoreClient(hiveConf), securityContext, subject),
                    hiveConf, securityContext, subject, hiveMetastore, hiveMetastoreProcesses);
    } else {
        return new HiveMetadataService(new HiveMetaStoreClient(hiveConf), hiveConf, securityContext, subject,
                hiveMetastore, hiveMetastoreProcesses);
    }
}
 
Example #4
Source File: HiveMetaStoreProxy.java    From griffin with Apache License 2.0 6 votes vote down vote up
@Bean
public IMetaStoreClient initHiveMetastoreClient() {
    HiveConf hiveConf = new HiveConf();
    hiveConf.set("hive.metastore.local", "false");
    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES,
        3);
    hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, uris);
    hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, attempts);
    hiveConf.setVar(HiveConf.ConfVars.HMSHANDLERINTERVAL, interval);
    try {
        client = HiveMetaStoreClient.newSynchronizedClient(new HiveMetaStoreClient(hiveConf));
    } catch (Exception e) {
        LOGGER.error("Failed to connect hive metastore. {}", e);
    }
    return client;
}
 
Example #5
Source File: HiveClientPool.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
protected HiveMetaStoreClient newClient()  {
  try {
    return new HiveMetaStoreClient(hiveConf);
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to connect to Hive Metastore");
  } catch (Throwable t) {
    if (t.getMessage().contains("Another instance of Derby may have already booted")) {
      throw new RuntimeMetaException(t, "Failed to start an embedded metastore because embedded " +
          "Derby supports only one client at a time. To fix this, use a metastore that supports " +
          "multiple clients.");
    }

    throw new RuntimeMetaException(t, "Failed to connect to Hive Metastore");
  }
}
 
Example #6
Source File: HiveServer2CoreTest.java    From beeju with Apache License 2.0 6 votes vote down vote up
@Test
public void dropTable() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  createUnpartitionedTable(DATABASE, tableName, server);

  try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
      Statement statement = connection.createStatement()) {
    String dropHql = String.format("DROP TABLE %s.%s", DATABASE, tableName);
    statement.execute(dropHql);
  }

  HiveMetaStoreClient client = server.getCore().newClient();
  try {
    client.getTable(DATABASE, tableName);
    fail(String.format("Table %s.%s was not deleted", DATABASE, tableName));
  } catch (NoSuchObjectException e) {
    // expected
  } finally {
    client.close();
  }
  server.shutdown();
}
 
Example #7
Source File: HiveServer2CoreTest.java    From beeju with Apache License 2.0 6 votes vote down vote up
@Test
public void dropDatabase() throws Exception {
  HiveServer2Core server = setupServer();
  String databaseName = "Another_DB";

  server.getCore().createDatabase(databaseName);
  try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
      Statement statement = connection.createStatement()) {
    String dropHql = String.format("DROP DATABASE %s", databaseName);
    statement.execute(dropHql);
  }

  HiveMetaStoreClient client = server.getCore().newClient();
  try {
    client.getDatabase(databaseName);
    fail(String.format("Database %s was not deleted", databaseName));
  } catch (NoSuchObjectException e) {
    // expected
  } finally {
    client.close();
  }
  server.shutdown();
}
 
Example #8
Source File: HiveServer2CoreTest.java    From beeju with Apache License 2.0 6 votes vote down vote up
@Test
public void addPartition() throws Exception {
  HiveServer2Core server = setupServer();
  String tableName = "my_table";
  createPartitionedTable(DATABASE, tableName, server);

  try (Connection connection = DriverManager.getConnection(server.getJdbcConnectionUrl());
      Statement statement = connection.createStatement()) {
    String addPartitionHql = String.format("ALTER TABLE %s.%s ADD PARTITION (partcol=1)", DATABASE, tableName);
    statement.execute(addPartitionHql);
  }

  HiveMetaStoreClient client = server.getCore().newClient();
  try {
    List<Partition> partitions = client.listPartitions(DATABASE, tableName, (short) -1);
    assertThat(partitions.size(), is(1));
    assertThat(partitions.get(0).getDbName(), is(DATABASE));
    assertThat(partitions.get(0).getTableName(), is(tableName));
    assertThat(partitions.get(0).getValues(), is(Arrays.asList("1")));
    assertThat(partitions.get(0).getSd().getLocation(),
        is(String.format("file:%s/%s/%s/partcol=1", server.getCore().tempDir(), DATABASE, tableName)));
  } finally {
    client.close();
  }
  server.shutdown();
}
 
Example #9
Source File: HiveServer2CoreTest.java    From beeju with Apache License 2.0 6 votes vote down vote up
private Table createUnpartitionedTable(String databaseName, String tableName, HiveServer2Core server)
    throws Exception {
  Table table = new Table();
  table.setDbName(databaseName);
  table.setTableName(tableName);
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example #10
Source File: HiveServer2CoreTest.java    From beeju with Apache License 2.0 6 votes vote down vote up
private Table createPartitionedTable(String databaseName, String tableName, HiveServer2Core server) throws Exception {
  Table table = new Table();
  table.setDbName(DATABASE);
  table.setTableName(tableName);
  table.setPartitionKeys(Arrays.asList(new FieldSchema("partcol", "int", null)));
  table.setSd(new StorageDescriptor());
  table.getSd().setCols(Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)));
  table.getSd().setInputFormat("org.apache.hadoop.mapred.TextInputFormat");
  table.getSd().setOutputFormat("org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
  table.getSd().setSerdeInfo(new SerDeInfo());
  table.getSd().getSerdeInfo().setSerializationLib("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe");
  HiveMetaStoreClient client = server.getCore().newClient();
  client.createTable(table);
  client.close();
  return table;
}
 
Example #11
Source File: ComparisonToolIntegrationTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws Exception {
  sourceTempUri = temporaryFolder.newFolder("source-temp");
  sourceWarehouseUri = temporaryFolder.newFolder("source-warehouse");
  sourceTableUri = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_TABLE);
  createSourceTable();

  replicaWarehouseUri = temporaryFolder.newFolder("replica-warehouse");
  replicaTableUri = new File(replicaWarehouseUri, DATABASE + "/" + REPLICA_TABLE);
  createReplicaTable();

  ymlFile = temporaryFolder.newFile("diff.yml");

  HiveMetaStoreClient client = catalog.client();
  LOG.info(">>>> Source Table = {}", client.getTable(DATABASE, SOURCE_TABLE));
  LOG.info(">>>> Replica Table = {}", client.getTable(DATABASE, REPLICA_TABLE));

  // Create hive config
  testHiveSiteXml = new HiveXml(catalog.getThriftConnectionUri(), catalog.connectionURL(), catalog.driverClassName())
      .create()
      .getCanonicalPath();

  outputFile = temporaryFolder.newFile("diff.out");
}
 
Example #12
Source File: HiveTableBaseTest.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException,
        TException,
        InvocationTargetException,
        NoSuchMethodException,
        IllegalAccessException,
        NoSuchFieldException, SQLException {
  this.executorService = Executors.newSingleThreadExecutor();
  hiveLocalDir = createTempDirectory("hive", asFileAttribute(fromString("rwxrwxrwx"))).toFile();
  setupDB("jdbc:derby:" + getDerbyPath() + ";create=true");

  this.server = thriftServer();
  executorService.submit(() -> server.serve());

  this.metastoreClient = new HiveMetaStoreClient(this.hiveConf);
  createIfNotExistsCatalog("hive");
  this.metastoreClient.createDatabase(new Database(DB_NAME, "description", getDBPath(), new HashMap<>()));
  new HiveTables(this.hiveConf).create(schema, partitionSpec, DB_NAME, TABLE_NAME);
}
 
Example #13
Source File: HiveDifferencesIntegrationTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws Exception {
  // We mock the checksum function because LocalFileSystem doesn't compute checksums
  when(checksumFunction.apply(any(Path.class))).thenAnswer(new Answer<String>() {
    @Override
    public String answer(InvocationOnMock invocation) throws Throwable {
      Path path = (Path) invocation.getArguments()[0];
      return path.getName();
    }
  });

  sourceWarehouseUri = temporaryFolder.newFolder("source-warehouse");
  sourceTableUri = new File(sourceWarehouseUri, DATABASE + "/" + SOURCE_TABLE);
  createTable(DATABASE, SOURCE_TABLE, sourceTableUri, null, null, false);

  replicaWarehouseUri = temporaryFolder.newFolder("replica-warehouse");
  replicaTableUri = new File(replicaWarehouseUri, DATABASE + "/" + REPLICA_TABLE);
  createTable(DATABASE, REPLICA_TABLE, replicaTableUri, DATABASE + "." + SOURCE_TABLE,
      sourceTableUri.toURI().toString(), true);

  HiveMetaStoreClient client = catalog.client();
  LOG.info(">>>> Source Table = {}", client.getTable(DATABASE, SOURCE_TABLE));
  LOG.info(">>>> Replica Table = {}", client.getTable(DATABASE, REPLICA_TABLE));
}
 
Example #14
Source File: TestUtils.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
static Table createPartitionedTable(HiveMetaStoreClient metaStoreClient, String database, String table, File location)
  throws Exception {

  Table hiveTable = new Table();
  hiveTable.setDbName(database);
  hiveTable.setTableName(table);
  hiveTable.setTableType(TableType.EXTERNAL_TABLE.name());
  hiveTable.putToParameters("EXTERNAL", "TRUE");

  hiveTable.setPartitionKeys(PARTITION_COLUMNS);

  StorageDescriptor sd = new StorageDescriptor();
  sd.setCols(DATA_COLUMNS);
  sd.setLocation(location.toURI().toString());
  sd.setParameters(new HashMap<>());
  sd.setSerdeInfo(new SerDeInfo());

  hiveTable.setSd(sd);

  metaStoreClient.createTable(hiveTable);

  return hiveTable;
}
 
Example #15
Source File: TestMetaStoreWithPigHCat.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  dataFile = new File(dataDir, SINGLE_TYPE_DATA_FILE_NAME);
  FileOutputStream to = new FileOutputStream(dataFile);
  Resources.copy(Resources.getResource(SINGLE_TYPE_DATA_FILE_NAME), to);
  to.close();

  policyFile = setAdminOnServer1(ADMINGROUP);
  policyFile
      .addRolesToGroup(USERGROUP1, db_all_role)
      .addRolesToGroup(USERGROUP2, "read_db_role")
      .addPermissionsToRole(db_all_role, "server=server1->db=" + dbName)
      .addPermissionsToRole("read_db_role",
          "server=server1->db=" + dbName + "->table=" + tabName2 + "->action=SELECT")
      .setUserGroupMapping(StaticUserGroup.getStaticMapping());
  writePolicyFile(policyFile);

  HiveMetaStoreClient client = context.getMetaStoreClient(ADMIN1);
  client.dropDatabase(dbName, true, true, true);
  createMetastoreDB(client, dbName);
  client.close();
}
 
Example #16
Source File: HiveAdapter.java    From hadoop-etl-udfs with MIT License 6 votes vote down vote up
static SchemaMetadata readMetadata(SchemaMetadataInfo schemaMetaInfo, ExaConnectionInformation connection, List<String> tableNames, String newSchema) throws SQLException, MetadataException {
    String databaseName = newSchema;
    if (newSchema == null) {
        databaseName = HiveAdapterProperties.getSchema(schemaMetaInfo.getProperties());
    }
    HiveMetaStoreClient hiveMetastoreClient = HiveMetastoreService.getHiveMetastoreClient(connection.getAddress(), HiveQueryGenerator.isKerberosAuth(connection.getPassword()), HiveAdapterProperties.getConnectionName(schemaMetaInfo.getProperties()));
    if (tableNames == null) {
        try {
            tableNames = hiveMetastoreClient.getAllTables(databaseName);
        } catch (MetaException e) {
            e.getMessage();
        }
    }
    List<TableMetadata> tables = new ArrayList<>();
    if (tableNames != null) {
        for (String tableName : tableNames) {
            Table table = HiveTableInformation.getHiveTable(hiveMetastoreClient, tableName, databaseName);
            tables.add(HiveTableInformation.getTableMetadataFromHCatTableMetadata(tableName.toUpperCase(), HiveTableInformation.getHCatTableMetadata(table)));
        }
    }
    return new SchemaMetadata("", tables);
}
 
Example #17
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void getDatabaseFromPatternPrefixed() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .overwriteConfigOnShutdown(false)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), "remote.?database")
      .federate("third", remoteServer.getThriftConnectionUri(), "no_match.*")
      .build();
  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  List<String> allDatabases = proxy.getAllDatabases();

  List<String> expected = Lists.newArrayList("default", LOCAL_DATABASE, PREFIXED_REMOTE_DATABASE);
  assertThat(allDatabases, is(expected));
}
 
Example #18
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void alterTableOnFederatedIsNotAllowedUsingManual() throws Exception {
  String[] mappableDatabases = new String[] { REMOTE_DATABASE };
  String[] writableDatabaseWhitelist = new String[] { REMOTE_DATABASE };
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.MANUAL)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .federate("doesNotMatterManualMode", remoteServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_ON_DATABASE_WHITELIST, mappableDatabases, writableDatabaseWhitelist)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  Table table = proxy.getTable(REMOTE_DATABASE, REMOTE_TABLE);
  Table newTable = new Table(table);
  newTable.setTableName("new_remote_table");

  proxy.alter_table_with_environmentContext(REMOTE_DATABASE, REMOTE_TABLE, newTable, null);
  assertThat(proxy.tableExists(REMOTE_DATABASE, "new_remote_table"), is(true));
}
 
Example #19
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void createDatabaseDatabaseUsingPrefixAndWhitelistingUpdates() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 
Example #20
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void createDatabaseUsingManualAndWhitelistingUpdatesConfig() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(),
          AccessControlType.READ_AND_WRITE_AND_CREATE_ON_DATABASE_WHITELIST)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // Should be allowed due to newly loaded write privileges
  proxy.alterDatabase("newDB", new Database(newDB));

  Federations federations = stopServerAndGetConfiguration();
  PrimaryMetaStore metaStore = federations.getPrimaryMetaStore();
  assertThat(metaStore.getWritableDatabaseWhiteList().size(), is(1));
  assertThat(metaStore.getWritableDatabaseWhiteList().get(0), is("newdb"));
  // Mapped databases should still map everything
  assertThat(metaStore.getMappedDatabases(), is(nullValue()));
}
 
Example #21
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void usePrefix() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri())
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = PREFIXED_REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example #22
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
@Test
public void typical() throws Exception {
  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri(), REMOTE_DATABASE)
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // Remote table
  String waggledRemoteDbName = REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, waggledRemoteDbName);
}
 
Example #23
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 6 votes vote down vote up
private void createRemoteTable(File tableUri) throws Exception {
  HiveMetaStoreClient client = remoteServer.client();

  Table hiveTable = createPartitionedTable(client, REMOTE_DATABASE, REMOTE_TABLE, tableUri);

  File partitionEurope = new File(tableUri, "continent=Europe");
  File partitionUk = new File(partitionEurope, "country=UK");

  File partitionAsia = new File(tableUri, "continent=Asia");
  File partitionChina = new File(partitionAsia, "country=China");

  LOG
      .info(">>>> Partitions added: {}",
          client
              .add_partitions(Arrays
                  .asList(newPartition(hiveTable, Arrays.asList("Europe", "UK"), partitionUk),
                      newPartition(hiveTable, Arrays.asList("Asia", "China"), partitionChina))));
}
 
Example #24
Source File: AbstractMetastoreTestWithStaticConfiguration.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
public Table makeMetastoreTableObject(HiveMetaStoreClient client,
    String dbName, String tabName, List<FieldSchema> cols) throws Exception {
  Table tbl = new Table();
  tbl.setDbName(dbName);
  tbl.setTableName(tabName);
  StorageDescriptor sd = new StorageDescriptor();
  tbl.setSd(sd);
  tbl.setParameters(new HashMap<String, String>());
  sd.setCols(cols);
  sd.setCompressed(false);
  sd.setParameters(new HashMap<String, String>());
  sd.setSerdeInfo(new SerDeInfo());
  sd.getSerdeInfo().setName(tbl.getTableName());
  sd.getSerdeInfo().setParameters(new HashMap<String, String>());
  sd.getSerdeInfo().getParameters()
      .put(serdeConstants.SERIALIZATION_FORMAT, "1");
  sd.setSortCols(new ArrayList<Order>());
  return tbl;
}
 
Example #25
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void usePrimaryPrefix() throws Exception {
  String primaryPrefix = "primary_";
  runner = WaggleDanceRunner
      .builder(configLocation)
      .databaseResolution(DatabaseResolution.PREFIXED)
      .primary("primary", localServer.getThriftConnectionUri(), READ_ONLY)
      .withPrimaryPrefix(primaryPrefix)
      .federate(SECONDARY_METASTORE_NAME, remoteServer.getThriftConnectionUri())
      .build();

  runWaggleDance(runner);
  HiveMetaStoreClient proxy = getWaggleDanceClient();

  // Local table
  String prefixedLocalDbName = primaryPrefix + LOCAL_DATABASE;
  Table localTable = proxy.getTable(prefixedLocalDbName, LOCAL_TABLE);
  assertThat(localTable.getDbName(), is(prefixedLocalDbName));

  // fetch without prefix works and result is prefixed
  Table localTable2 = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(localTable2.getDbName(), is(prefixedLocalDbName));

  // Remote table
  String prefixedRemoteDbName = PREFIXED_REMOTE_DATABASE;
  assertTypicalRemoteTable(proxy, prefixedRemoteDbName);
}
 
Example #26
Source File: HiveClientPool.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@Override
protected HiveMetaStoreClient reconnect(HiveMetaStoreClient client) {
  try {
    client.close();
    client.reconnect();
  } catch (MetaException e) {
    throw new RuntimeMetaException(e, "Failed to reconnect to Hive Metastore");
  }
  return client;
}
 
Example #27
Source File: WaggleDanceIntegrationTest.java    From waggle-dance with Apache License 2.0 5 votes vote down vote up
@Test
public void readWriteCreateAllowed() throws Exception {
  String writableDatabase = "writable_db";

  localServer.createDatabase(writableDatabase);

  runner = WaggleDanceRunner
      .builder(configLocation)
      .primary("primary", localServer.getThriftConnectionUri(), AccessControlType.READ_AND_WRITE_AND_CREATE)
      .build();

  runWaggleDance(runner);

  HiveMetaStoreClient proxy = getWaggleDanceClient();
  // create rights
  proxy.createDatabase(new Database("newDB", "", new File(localWarehouseUri, "newDB").toURI().toString(), null));
  Database newDB = proxy.getDatabase("newDB");
  assertNotNull(newDB);

  // read rights
  Table localTable = localServer.client().getTable(LOCAL_DATABASE, LOCAL_TABLE);
  Table waggledLocalTable = proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
  assertThat(waggledLocalTable, is(localTable));

  // write rights
  proxy.dropTable(LOCAL_DATABASE, LOCAL_TABLE);
  try {
    proxy.getTable(LOCAL_DATABASE, LOCAL_TABLE);
    fail("Should get NoSuchObjectException");
  } catch (NoSuchObjectException e) {
    // Local table should be allowed to drop, so it now no longer exists and we get an appropriate exception
  }
}
 
Example #28
Source File: HiveMetastoreService.java    From hadoop-etl-udfs with MIT License 5 votes vote down vote up
public static HiveMetaStoreClient checkHiveMetaStoreClient(String hiveMetastoreUrl,boolean useKerberos, String hcatUserOrServicePrincipal) throws MetaException {
    HiveConf hiveConf = new HiveConf(new Configuration(), HiveConf.class);
    hiveConf.set("hive.metastore.local", "false");
    hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUrl);
    hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3);
    if (useKerberos) {
        System.out.println("Add hive.metastore.kerberos.principal: " + hcatUserOrServicePrincipal);
        hiveConf.set("hive.metastore.kerberos.principal", hcatUserOrServicePrincipal);
        hiveConf.set("hive.metastore.sasl.enabled", "true");
    }
    return new HiveMetaStoreClient(hiveConf);
}
 
Example #29
Source File: Context.java    From incubator-sentry with Apache License 2.0 5 votes vote down vote up
public HiveMetaStoreClient getMetaStoreClient(String userName) throws Exception {
  UserGroupInformation clientUgi = UserGroupInformation.createRemoteUser(userName);
  HiveMetaStoreClient client = (HiveMetaStoreClient) clientUgi.
      doAs(new PrivilegedExceptionAction<Object> () {
        @Override
        public HiveMetaStoreClient run() throws Exception {
          return new HiveMetaStoreClient(new HiveConf());
        }
      });
  return client;
}
 
Example #30
Source File: HiveMetastoreTest.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void startMetastore() throws Exception {
  HiveMetastoreTest.metastore = new TestHiveMetastore();
  metastore.start();
  HiveMetastoreTest.hiveConf = metastore.hiveConf();
  HiveMetastoreTest.metastoreClient = new HiveMetaStoreClient(hiveConf);
  String dbPath = metastore.getDatabasePath(DB_NAME);
  Database db = new Database(DB_NAME, "description", dbPath, new HashMap<>());
  metastoreClient.createDatabase(db);
  HiveMetastoreTest.catalog = new HiveCatalog(hiveConf);
}