Java Code Examples for org.apache.hadoop.hbase.client.Connection#getAdmin()

public static void main(String[] args) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
  // tag::MERGE1[]
  Configuration conf = HBaseConfiguration.create();
  Connection connection = ConnectionFactory.createConnection(conf);
  HBaseAdmin admin = (HBaseAdmin)connection.getAdmin();
  List<HRegionInfo> regions = admin.getTableRegions(TableName.valueOf("t1")); //<1>
  LOG.info("testtable contains " + regions.size() + " regions.");
  for (int index = 0; index < regions.size() / 2; index++) {
    HRegionInfo region1 = regions.get(index*2);
    HRegionInfo region2 = regions.get(index*2+1);
    LOG.info("Merging regions " + region1 + " and " + region2);
    admin.mergeRegions(region1.getEncodedNameAsBytes(), 
                       region2.getEncodedNameAsBytes(), false); //<2>
  }
  admin.close();
  // end::MERGE1[]
}
 

private static void startThirdCluster() throws Exception {
  LOG.info("Setup Zk to same one from UTIL1 and UTIL2");
  UTIL3.setZkCluster(UTIL1.getZkCluster());
  UTIL3.startMiniCluster(NUM_SLAVES1);

  TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName)
      .setMobEnabled(true)
      .setMobThreshold(4000)
      .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();

  Connection connection3 = ConnectionFactory.createConnection(CONF3);
  try (Admin admin3 = connection3.getAdmin()) {
    admin3.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  UTIL3.waitUntilAllRegionsAssigned(tableName);
  htable3 = connection3.getTable(tableName);
}
 

private static void show() throws IOException {
    Map<String, List<String>> envs = Maps.newHashMap();

    // get all kylin hbase tables
    KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
    Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
    Admin hbaseAdmin = conn.getAdmin();
    String tableNamePrefix = kylinConfig.getHBaseTableNamePrefix();
    HTableDescriptor[] tableDescriptors = hbaseAdmin.listTables(tableNamePrefix + ".*");
    for (HTableDescriptor desc : tableDescriptors) {
        String host = desc.getValue(IRealizationConstants.HTableTag);
        if (StringUtils.isEmpty(host)) {
            add("unknown", desc.getNameAsString(), envs);
        } else {
            add(host, desc.getNameAsString(), envs);
        }
    }

    for (Map.Entry<String, List<String>> entry : envs.entrySet()) {
        System.out.println(entry.getKey() + " has htable count: " + entry.getValue().size());
    }
    hbaseAdmin.close();
}
 

private void alter() throws IOException {
    Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
    Admin hbaseAdmin = null;

    try {
        hbaseAdmin = conn.getAdmin();
        HTableDescriptor table = hbaseAdmin.getTableDescriptor(TableName.valueOf(tableName));

        hbaseAdmin.disableTable(table.getTableName());
        table.setValue(metadataKey, metadataValue);
        hbaseAdmin.modifyTable(table.getTableName(), table);
        hbaseAdmin.enableTable(table.getTableName());
    } finally {
        if (hbaseAdmin != null) {
            hbaseAdmin.close();
        }
    }
}
 

private void clean() throws IOException {
    Connection conn = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl());
    Admin hbaseAdmin = conn.getAdmin();

    for (HTableDescriptor descriptor : hbaseAdmin.listTables()) {
        String name = descriptor.getNameAsString().toLowerCase(Locale.ROOT);
        if (name.startsWith("kylin") || name.startsWith("_kylin")) {
            String x = descriptor.getValue(IRealizationConstants.HTableTag);
            System.out.println("table name " + descriptor.getNameAsString() + " host: " + x);
            System.out.println(descriptor);
            System.out.println();

            descriptor.setValue(IRealizationConstants.HTableOwner, "[email protected]");
            hbaseAdmin.modifyTable(TableName.valueOf(descriptor.getNameAsString()), descriptor);
        }
    }
    hbaseAdmin.close();
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TestReplicationBase.setUpBeforeClass();

  TableDescriptor peerTable = TableDescriptorBuilder.newBuilder(peerTableName).setColumnFamily(
                  ColumnFamilyDescriptorBuilder.newBuilder(noRepfamName).setMaxVersions(100)
                          .build()).build();

  Connection connection2 = ConnectionFactory.createConnection(CONF2);
  try (Admin admin2 = connection2.getAdmin()) {
    admin2.createTable(peerTable, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  htable3 = connection2.getTable(peerTableName);
}
 

private static void dropTable() {
  Connection conn = BigtableConfiguration.connect(projectID, instanceID);
  try {
    Admin admin = conn.getAdmin();
    admin.disableTable(TableName.valueOf(tableName));
    admin.deleteTable(TableName.valueOf(tableName));
  } catch (Exception ex) {
    LOG.info("dropTable(): exception {} ", ex.toString());
  }
}
 

/**
 * Allows subclasses to initialize the table information.
 *
 * @param connection  The Connection to the HBase cluster. MUST be unmanaged. We will close.
 * @param tableName  The {@link TableName} of the table to process.
 * @throws IOException
 */
protected void initializeTable(Connection connection, TableName tableName) throws IOException {
  if (this.table != null || this.connection != null) {
    LOG.warn("initializeTable called multiple times. Overwriting connection and table " +
        "reference; TableInputFormatBase will not close these old references when done.");
  }
  this.table = connection.getTable(tableName);
  this.regionLocator = connection.getRegionLocator(tableName);
  this.admin = connection.getAdmin();
  this.connection = connection;
}
 

/**
 * List all the userPermissions matching the given table pattern and user name.
 * @param connection Connection
 * @param tableRegex The regular expression string to match against
 * @param userName User name, if empty then all user permissions will be retrieved.
 * @return List of UserPermissions
 * @throws Throwable on failure
 */
public static List<UserPermission> getUserPermissions(Connection connection, String tableRegex,
    String userName) throws Throwable {
  List<UserPermission> permList = new ArrayList<>();
  try (Admin admin = connection.getAdmin()) {
    if (tableRegex == null || tableRegex.isEmpty()) {
      permList = admin.getUserPermissions(
        GetUserPermissionsRequest.newBuilder().withUserName(userName).build());
    } else if (tableRegex.charAt(0) == '@') { // Namespaces
      String namespaceRegex = tableRegex.substring(1);
      for (NamespaceDescriptor nsds : admin.listNamespaceDescriptors()) { // Read out all
                                                                          // namespaces
        String namespace = nsds.getName();
        if (namespace.matches(namespaceRegex)) { // Match the given namespace regex?
          permList.addAll(admin.getUserPermissions(
            GetUserPermissionsRequest.newBuilder(namespace).withUserName(userName).build()));
        }
      }
    } else { // Tables
      List<TableDescriptor> htds = admin.listTableDescriptors(Pattern.compile(tableRegex), true);
      for (TableDescriptor htd : htds) {
        permList.addAll(admin.getUserPermissions(GetUserPermissionsRequest
            .newBuilder(htd.getTableName()).withUserName(userName).build()));
      }
    }
  }
  return permList;
}
 

/**
 * Test a table creation including a coprocessor path
 * which is not whitelisted. Table will not be created due to the
 * offending coprocessor.
 */
@Test
public void testCreationNonWhitelistedCoprocessorPath() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  // load coprocessor under test
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      CoprocessorWhitelistMasterObserver.class.getName());
  conf.setStrings(CoprocessorWhitelistMasterObserver.CP_COPROCESSOR_WHITELIST_PATHS_KEY,
      new String[]{});
  // set retries low to raise exception quickly
  conf.setInt("hbase.client.retries.number", 5);
  UTIL.startMiniCluster();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TEST_TABLE);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY);
  tableDescriptor.setColumnFamily(familyDescriptor);
  tableDescriptor.setCoprocessor(
    CoprocessorDescriptorBuilder.newBuilder("net.clayb.hbase.coprocessor.NotWhitelisted")
      .setJarPath("file:///notpermitted/couldnotpossiblyexist.jar")
      .setPriority(Coprocessor.PRIORITY_USER)
      .setProperties(Collections.emptyMap())
      .build());
  Connection connection = ConnectionFactory.createConnection(conf);
  Admin admin = connection.getAdmin();
  LOG.info("Creating Table");
  try {
    admin.createTable(tableDescriptor);
    fail("Expected coprocessor to raise IOException");
  } catch (IOException e) {
    // swallow exception from coprocessor
  }
  LOG.info("Done Creating Table");
  // ensure table was not created
  assertEquals(0,
    admin.listTableDescriptors(Pattern.compile("^" + TEST_TABLE.getNameAsString() + "$")).size());
}
 

@Before
	public void setUp() throws IOException {
		Configuration conf = HBaseConfiguration.create();
//		conf.set("hbase.master", "node-02:16010;node-03:16010");
		conf.set("hbase.zookeeper.quorum", "node-01:2181,node-02:2181,node-03:2181");
		Connection connection = ConnectionFactory.createConnection(conf);
		admin = connection.getAdmin();
	}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TestReplicationBase.setUpBeforeClass();
  TableDescriptor peerTable = TableDescriptorBuilder.newBuilder(peerTableName).setColumnFamily(
                  ColumnFamilyDescriptorBuilder.newBuilder(noRepfamName).setMaxVersions(100)
                          .build()).build();
  Connection connection2 = ConnectionFactory.createConnection(CONF2);
  try (Admin admin2 = connection2.getAdmin()) {
    admin2.createTable(peerTable, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  htable3 = connection2.getTable(peerTableName);
}
 

private static void startClusters() throws Exception {
  UTIL1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = UTIL1.getZkCluster();
  LOG.info("Setup first Zk");

  UTIL2.setZkCluster(miniZK);
  LOG.info("Setup second Zk");

  CONF_WITH_LOCALFS = HBaseConfiguration.create(CONF1);
  UTIL1.startMiniCluster(NUM_SLAVES1);
  // Have a bunch of slave servers, because inter-cluster shipping logic uses number of sinks
  // as a component in deciding maximum number of parallel batches to send to the peer cluster.
  UTIL2.startMiniCluster(NUM_SLAVES2);

  hbaseAdmin = ConnectionFactory.createConnection(CONF1).getAdmin();

  TableDescriptor table = TableDescriptorBuilder.newBuilder(tableName)
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName).setMaxVersions(100)
          .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();

  Connection connection1 = ConnectionFactory.createConnection(CONF1);
  Connection connection2 = ConnectionFactory.createConnection(CONF2);
  try (Admin admin1 = connection1.getAdmin()) {
    admin1.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  try (Admin admin2 = connection2.getAdmin()) {
    admin2.createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  }
  UTIL1.waitUntilAllRegionsAssigned(tableName);
  UTIL2.waitUntilAllRegionsAssigned(tableName);
  htable1 = connection1.getTable(tableName);
  htable2 = connection2.getTable(tableName);
}
 

/**
 *
 * @param sourceTableName
 * @param sourceTable
 * @param kylinConfig
 * @return Pair of HTableName and shard number
 * @throws IOException
 */
private Pair<String, Integer> createHTable(String sourceTableName, IReadableTable sourceTable,
        KylinConfig kylinConfig) throws IOException {
    TableSignature signature = sourceTable.getSignature();
    int shardNum = calculateShardNum(kylinConfig, signature.getSize());
    Connection conn = getHBaseConnection(kylinConfig);
    Admin admin = conn.getAdmin();
    String hTableName = genHTableName(kylinConfig, admin, sourceTableName);

    TableName tableName = TableName.valueOf(hTableName);
    HTableDescriptor hTableDesc = new HTableDescriptor(tableName);
    hTableDesc.setCompactionEnabled(false);
    hTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
    hTableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
    hTableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
    String commitInfo = KylinVersion.getGitCommitInfo();
    if (!StringUtils.isEmpty(commitInfo)) {
        hTableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
    }

    HColumnDescriptor cf = CubeHTableUtil.createColumnFamily(kylinConfig, HBaseLookupRowEncoder.CF_STRING, false);
    hTableDesc.addFamily(cf);

    try {
        if (shardNum > 1) {
            admin.createTable(hTableDesc, getSplitsByShardNum(shardNum));
        } else {
            admin.createTable(hTableDesc);
        }
    } finally {
        IOUtils.closeQuietly(admin);
    }
    return new Pair<>(hTableName, shardNum);
}
 

@VisibleForTesting
HRegionFileSystem getFileSystem(Connection connection) throws IOException {
  Admin admin = connection.getAdmin();
  return HRegionFileSystem.openRegionFromFileSystem(admin.getConfiguration(),
    CommonFSUtils.getCurrentFileSystem(admin.getConfiguration()), CommonFSUtils.getTableDir(
      CommonFSUtils.getRootDir(admin.getConfiguration()), region.getTable()),
    region, true);
}
 

public static boolean tableExists(Connection conn, String tableName) throws IOException {
    Admin hbase = conn.getAdmin();
    try {
        return hbase.tableExists(TableName.valueOf(tableName));
    } finally {
        hbase.close();
    }
}
 

public static boolean isAccessControllerRunning(Connection connection)
    throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
  try (Admin admin = connection.getAdmin()) {
    return admin.isTableAvailable(ACL_TABLE_NAME);
  }
}
 

/**
 * Computes size of each region for table and given column families.
 * */
public HBaseRegionSizeCalculator(String tableName, Connection hbaseConnection) throws IOException {

    Table table = null;
    Admin admin = null;
    try {
        table = hbaseConnection.getTable(TableName.valueOf(tableName));
        admin = hbaseConnection.getAdmin();

        if (!enabled(table.getConfiguration())) {
            logger.info("Region size calculation disabled.");
            return;
        }

        logger.info("Calculating region sizes for table \"" + table.getName() + "\".");

        // Get regions for table.
        RegionLocator regionLocator = hbaseConnection.getRegionLocator(table.getName());
        List<HRegionLocation> regionLocationList = regionLocator.getAllRegionLocations();
        Set<byte[]> tableRegions = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);

        for (HRegionLocation hRegionLocation : regionLocationList) {
            tableRegions.add(hRegionLocation.getRegionInfo().getRegionName());
        }

        ClusterStatus clusterStatus = admin.getClusterStatus();
        Collection<ServerName> servers = clusterStatus.getServers();
        final long megaByte = 1024L * 1024L;

        // Iterate all cluster regions, filter regions from our table and
        // compute their size.
        for (ServerName serverName : servers) {
            ServerLoad serverLoad = clusterStatus.getLoad(serverName);

            for (RegionLoad regionLoad : serverLoad.getRegionsLoad().values()) {
                byte[] regionId = regionLoad.getName();

                if (tableRegions.contains(regionId)) {

                    long regionSizeBytes = regionLoad.getStorefileSizeMB() * megaByte;
                    sizeMap.put(regionId, regionSizeBytes);
                    countMap.put(regionId, new Pair<>(regionLoad.getStores(), regionLoad.getStorefiles()));

                    if (regionSizeBytes == 0L) {
                        logger.info("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
                    }
                }
            }
        }
    } finally {
        IOUtils.closeQuietly(admin);
    }

}
 

@Test
public void testNamespaceSpaceAndRPCQuotaRemoved() throws Exception {
  final Connection conn = TEST_UTIL.getConnection();
  final Admin admin = conn.getAdmin();
  final String ns = testName.getMethodName();
  // Drop the ns if it somehow exists
  if (namespaceExists(ns)) {
    admin.deleteNamespace(ns);
  }

  // Create the ns
  NamespaceDescriptor desc = NamespaceDescriptor.create(ns).build();
  admin.createNamespace(desc);

  assertEquals(0, getNumSpaceQuotas());
  assertEquals(0, getThrottleQuotas());

  // Set Both quotas
  QuotaSettings settings =
      QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
  admin.setQuota(settings);

  settings =
      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
  admin.setQuota(settings);

  assertEquals(1, getNumSpaceQuotas());
  assertEquals(1, getThrottleQuotas());

  // Remove Space quota
  settings = QuotaSettingsFactory.removeNamespaceSpaceLimit(ns);
  admin.setQuota(settings);
  assertEquals(0, getNumSpaceQuotas());
  assertEquals(1, getThrottleQuotas());

  // Set back the space quota
  settings = QuotaSettingsFactory.limitNamespaceSpace(ns, 1024L, SpaceViolationPolicy.NO_INSERTS);
  admin.setQuota(settings);
  assertEquals(1, getNumSpaceQuotas());
  assertEquals(1, getThrottleQuotas());

  // Remove the throttle quota
  settings = QuotaSettingsFactory.unthrottleNamespace(ns);
  admin.setQuota(settings);
  assertEquals(1, getNumSpaceQuotas());
  assertEquals(0, getThrottleQuotas());

  // Set back the throttle quota
  settings =
      QuotaSettingsFactory.throttleNamespace(ns, ThrottleType.REQUEST_SIZE, 2L, TimeUnit.HOURS);
  admin.setQuota(settings);
  assertEquals(1, getNumSpaceQuotas());
  assertEquals(1, getThrottleQuotas());

  // Delete the namespace and check that both the quotas have been dropped as well
  admin.deleteNamespace(ns);

  assertEquals(0, getNumSpaceQuotas());
  assertEquals(0, getThrottleQuotas());
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
    MR_TEST_UTIL = new MRTestUtil(HBASE_TEST_UTILITY);
    HBASE_TEST_UTILITY.startMiniCluster();
    MR_TEST_UTIL.startMrCluster();
    
    FileSystem fs = FileSystem.get(HBASE_TEST_UTILITY.getConfiguration());
    int zkClientPort = HBASE_TEST_UTILITY.getZkCluster().getClientPort();
    
    SOLR_TEST_UTILITY = new SolrTestingUtility(zkClientPort, NetUtils.getFreePort(),
            ImmutableMap.of(
                    "solr.hdfs.blockcache.enabled", "false",
                    "solr.directoryFactory", "HdfsDirectoryFactory",
                    "solr.hdfs.home", fs.makeQualified(new Path("/solrdata")).toString()));
    SOLR_TEST_UTILITY.start();
    
    SOLR_TEST_UTILITY.uploadConfig("config1", new File(MINIMR_CONF_DIR, "conf"));
    SOLR_TEST_UTILITY.createCollection("collection1", "config1", 2);
    SOLR_TEST_UTILITY.createCollection("collection2", "config1", 2);

    COLLECTION1 = new CloudSolrClient.Builder().withZkHost(SOLR_TEST_UTILITY.getZkConnectString()).build();        
    COLLECTION1.setDefaultCollection("collection1");

    SOLR_ZK = "127.0.0.1:" + zkClientPort + "/solr";
    INDEXER_ZK = "localhost:" + zkClientPort;
    ZooKeeperItf zkItf = ZkUtil.connect(INDEXER_ZK, 15000);
    INDEXER_MODEL = new IndexerModelImpl(zkItf, "/ngdata/hbaseindexer");
    IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                                            .name("zkindexerdef")
                                            .indexerComponentFactory(DefaultIndexerComponentFactory.class.getName())
                                            .configuration(Resources.toByteArray(Resources.getResource(
                                                    HBaseMapReduceIndexerToolGoLiveTest.class, "user_indexer.xml")))
                                            .connectionParams(ImmutableMap.of(
                                                    "solr.zk", SOLR_ZK,
                                                    "solr.collection", "collection1"))
                                            .build();

    addAndWaitForIndexer(indexerDef);
    
    Closer.close(zkItf);
    
    HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(TEST_TABLE_NAME));
    tableDescriptor.addFamily(new HColumnDescriptor(TEST_COLFAM_NAME));
    Connection connection = ConnectionFactory.createConnection(HBASE_TEST_UTILITY.getConfiguration());
    HBASE_ADMIN = connection.getAdmin();
    HBASE_ADMIN.createTable(tableDescriptor, new byte[][]{Bytes.toBytes("row0800"), Bytes.toBytes("row1600")});
    
    RECORD_TABLE = connection.getTable(TableName.valueOf(TEST_TABLE_NAME)); 
    
    for (int i = 0; i < RECORD_COUNT; i++) {
        writeHBaseRecord(String.format("row%04d", i), ImmutableMap.of(
                "firstname", String.format("John%04d", i),
                "lastname", String.format("Doe%04d", i)));
    }
    
    
}