Java Code Examples for org.apache.hadoop.hbase.client.TableDescriptorBuilder#ModifyableTableDescriptor

private void createTable() throws Exception {
  deleteTable();
  LOG.info("Creating table");
  Configuration conf = util.getConfiguration();
  String encodingKey = String.format(ENCODING_KEY, this.getClass().getSimpleName());
  DataBlockEncoding blockEncoding = DataBlockEncoding.valueOf(conf.get(encodingKey, "FAST_DIFF"));
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);
  for (byte[] cf : dataGen.getColumnFamilies()) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(cf);
    familyDescriptor.setDataBlockEncoding(blockEncoding);
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  int serverCount = util.getHBaseClusterInterface().getClusterMetrics()
    .getLiveServerMetrics().size();
  byte[][] splits = new RegionSplitter.HexStringSplit().split(serverCount * REGIONS_PER_SERVER);
  util.getAdmin().createTable(tableDescriptor, splits);
  LOG.info("Created table");
}
 

private static void changeConstraintEnabled(
    TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor,
  Class<? extends Constraint> clazz, boolean enabled) throws IOException {
  // get the original constraint
  Pair<String, String> entry = getKeyValueForClass(tableDescriptor, clazz);
  if (entry == null) {
    throw new IllegalArgumentException("Constraint: " + clazz.getName()
      + " is not associated with this table. You can't enable it!");
  }

  // create a new configuration from that conf
  Configuration conf = readConfiguration(entry.getSecond());

  // set that it is enabled
  conf.setBoolean(ENABLED_KEY, enabled);

  // write it back out
  writeConstraint(tableDescriptor, entry.getFirst(), conf);
}
 

public void init(Configuration conf, long numRows) throws IOException {
  this.conf = conf;
  this.count = numRows;
  initConf();
  printConf();
  tableDescriptor = new TableDescriptorBuilder.ModifyableTableDescriptor(
    TableName.valueOf("testMobCompactTable"));
  Connection conn = ConnectionFactory.createConnection(this.conf);
  this.admin = conn.getAdmin();
  this.familyDescriptor = new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(fam);
  this.familyDescriptor.setMobEnabled(true);
  this.familyDescriptor.setMobThreshold(mobLen);
  this.familyDescriptor.setMaxVersions(1);
  this.tableDescriptor.setColumnFamily(familyDescriptor);
  if (admin.tableExists(tableDescriptor.getTableName())) {
    admin.disableTable(tableDescriptor.getTableName());
    admin.deleteTable(tableDescriptor.getTableName());
  }
  admin.createTable(tableDescriptor);
  table = conn.getTable(tableDescriptor.getTableName());
}
 

HRegion initHRegion(byte[] tableName, String callingMethod, Configuration conf,
    byte[]... families) throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : families) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  ChunkCreator.initialize(MemStoreLABImpl.CHUNK_SIZE_DEFAULT, false, 0, 0, 0, null);
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  Path path = new Path(DIR + callingMethod);
  WAL wal = HBaseTestingUtility.createWal(conf, path, info);
  HRegion r = HRegion.createHRegion(info, path, conf, tableDescriptor, wal);
  // this following piece is a hack. currently a coprocessorHost
  // is secretly loaded at OpenRegionHandler. we don't really
  // start a region server here, so just manually create cphost
  // and set it to region.
  RegionCoprocessorHost host = new RegionCoprocessorHost(r, null, conf);
  r.setCoprocessorHost(host);
  return r;
}
 

@Test
public void testLoadAndVerify() throws Exception {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(TEST_NAME));

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY));

  Admin admin = getTestingUtil(getConf()).getAdmin();
  admin.createTable(tableDescriptor, Bytes.toBytes(0L), Bytes.toBytes(-1L), 40);

  doLoad(getConf(), tableDescriptor);
  doVerify(getConf(), tableDescriptor);

  // Only disable and drop if we succeeded to verify - otherwise it's useful
  // to leave it around for post-mortem
  getTestingUtil(getConf()).deleteTable(tableDescriptor.getTableName());
}
 

@Test
public void testPriorityRegionIsOpenedWithSeparateThreadPool() throws Exception {
  final TableName tableName = TableName.valueOf(TestRegionOpen.class.getSimpleName());
  ThreadPoolExecutor exec = getRS().getExecutorService()
      .getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION);
  long completed = exec.getCompletedTaskCount();

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  tableDescriptor.setPriority(HConstants.HIGH_QOS);
  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
      HConstants.CATALOG_FAMILY));
  try (Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
      Admin admin = connection.getAdmin()) {
    admin.createTable(tableDescriptor);
  }

  assertEquals(completed + 1, exec.getCompletedTaskCount());
}
 

/**
 * Create a table with specified table name and region number.
 * @param tableName the name of the table to be created
 * @param regionNum number of regions to create
 * @throws IOException
 */
private static void createTable(TableName tableName, int regionNum)
    throws IOException {
  int expectedRegions = regionNum;
  byte[][] splitKeys = new byte[expectedRegions - 1][];
  for (int i = 1; i < expectedRegions; i++) {
    byte splitKey = (byte) i;
    splitKeys[i - 1] = new byte[] { splitKey, splitKey, splitKey };
  }

  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
      HConstants.CATALOG_FAMILY));
  admin.createTable(tableDescriptor, splitKeys);

  try (RegionLocator r = CONNECTION.getRegionLocator(tableName)) {
    List<HRegionLocation> regions = r.getAllRegionLocations();
    assertEquals("Tried to create " + expectedRegions + " regions "
        + "but only found " + regions.size(), expectedRegions, regions.size());
  }
}
 

public HTableDescriptor createTableDescriptor(final TableName tableName,
    byte[][] families, int maxVersions) {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);

  for (byte[] family : families) {
    ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family)
        .setMaxVersions(maxVersions);
    if (isNewVersionBehaviorEnabled()) {
      familyDescriptor.setNewVersionBehavior(true);
    }
    tableDescriptor.setColumnFamily(familyDescriptor);
  }
  return new HTableDescriptor(tableDescriptor);
}
 

private Table createTable() throws IOException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY);
  familyDescriptor.setNewVersionBehavior(true);
  familyDescriptor.setMaxVersions(3);
  tableDescriptor.setColumnFamily(familyDescriptor);
  TEST_UTIL.getAdmin().createTable(tableDescriptor);
  return TEST_UTIL.getConnection().getTable(tableName);
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  UTIL.getConfiguration().set("hbase.client.retries.number", "3");
  UTIL.startMiniCluster();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableAname));
  for (ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor family : families) {
    tableDescriptor.setColumnFamily(family);
  }
  try (Admin admin = UTIL.getAdmin()) {
    admin.createTable(tableDescriptor);
  }
}
 

private static void updateTable(
    final TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor) throws Exception {
  Admin admin = TEST_UTIL.getAdmin();
  admin.disableTable(tableDescriptor.getTableName());
  admin.modifyTable(tableDescriptor);
  admin.enableTable(tableDescriptor.getTableName());
}
 

/**
 * Check to make sure a constraint is unloaded when it fails
 * @throws Exception
 */
@Test
public void testIsUnloaded() throws Exception {
  // create the table
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);

  // add a family to the table
  for (byte[] family : new byte[][]{dummy, test}) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  // make sure that constraints are unloaded
  Constraints.add(tableDescriptor, RuntimeFailConstraint.class);
  // add a constraint to check to see if is run
  Constraints.add(tableDescriptor, CheckWasRunConstraint.class);
  CheckWasRunConstraint.wasRun = false;

  util.getAdmin().createTable(tableDescriptor);
  Table table = util.getConnection().getTable(tableName);

  // test that we do fail on violation
  Put put = new Put(row1);
  byte[] qualifier = new byte[0];
  put.addColumn(dummy, qualifier, Bytes.toBytes("pass"));

  try{
  table.put(put);
  fail("RuntimeFailConstraint wasn't triggered - this put shouldn't work!");
  } catch (Exception e) {// NOOP
  }

  // try the put again, this time constraints are not used, so it works
  table.put(put);
  // and we make sure that constraints were not run...
  assertFalse(CheckWasRunConstraint.wasRun);
  table.close();
}
 

public static void enable(TableDescriptorBuilder.ModifyableTableDescriptor desc)
    throws IOException {
  // if the CP has already been loaded, do nothing
  String clazz = ConstraintProcessor.class.getName();
  if (desc.hasCoprocessor(clazz)) {
    return;
  }

  // add the constrain processor CP to the table
  desc.setCoprocessor(clazz);
}
 

private void runCoprocessorConnectionToRemoteTable(Class clazz, boolean[] completeCheck)
    throws Throwable {
  // Check if given class implements RegionObserver.
  assert(RegionObserver.class.isAssignableFrom(clazz));
  TableDescriptorBuilder.ModifyableTableDescriptor primaryDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(primaryTable);

  primaryDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  // add our coprocessor
  primaryDescriptor.setCoprocessor(clazz.getName());

  TableDescriptorBuilder.ModifyableTableDescriptor otherDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(otherTable);

  otherDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));


  Admin admin = UTIL.getAdmin();
  admin.createTable(primaryDescriptor);
  admin.createTable(otherDescriptor);

  Table table = UTIL.getConnection().getTable(TableName.valueOf("primary"));
  Put p = new Put(new byte[] { 'a' });
  p.addColumn(family, null, new byte[]{'a'});
  table.put(p);
  table.close();

  Table target = UTIL.getConnection().getTable(otherTable);
  assertTrue("Didn't complete update to target table!", completeCheck[0]);
  assertEquals("Didn't find inserted row", 1, getKeyValueCount(target));
  target.close();
}
 

private TableDescriptorBuilder.ModifyableTableDescriptor createTableDescriptor(
    TableName tableName, boolean hasMob) {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY);
  if (hasMob) {
    familyDescriptor.setMobEnabled(true);
    familyDescriptor.setMobThreshold(0);
  }
  tableDescriptor.setColumnFamily(familyDescriptor);
  return tableDescriptor;
}
 

@Test
public void testAddSameColumnFamilyTwice() throws IOException {
  Admin admin = TEST_UTIL.getAdmin();
  // Create a table with one families
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TABLE_NAME);

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY_0));
  admin.createTable(tableDescriptor);
  admin.disableTable(TABLE_NAME);
  try {
    // Verify the table descriptor
    verifyTableDescriptor(TABLE_NAME, FAMILY_0);

    // Modify the table removing one family and verify the descriptor
    admin.addColumnFamily(TABLE_NAME,
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY_1));
    verifyTableDescriptor(TABLE_NAME, FAMILY_0, FAMILY_1);

    try {
      // Add same column family again - expect failure
      admin.addColumnFamily(TABLE_NAME,
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY_1));
      Assert.fail("Delete a non-exist column family should fail");
    } catch (InvalidFamilyOperationException e) {
      // Expected.
    }

  } finally {
    admin.deleteTable(TABLE_NAME);
  }
}
 

private HRegion initHRegion(byte [] tableName, String callingMethod, byte[] ... families)
    throws IOException {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));
  for(byte [] family : families) {
    tableDescriptor.setColumnFamily(
      new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
  }
  RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  return HBTU.createLocalHRegion(info, tableDescriptor);
}
 

protected void createTable(TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor)
    throws Exception {
  deleteTable();
  if (util.getHBaseClusterInterface() instanceof MiniHBaseCluster) {
    LOG.warn("Test does not make a lot of sense for minicluster. Will set flush size low.");
    tableDescriptor.setValue(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, "1048576");
  }
  byte[][] splits = new RegionSplitter.HexStringSplit().split(
      util.getHBaseClusterInterface().getClusterMetrics().getLiveServerMetrics().size());
  util.getAdmin().createTable(tableDescriptor, splits);
}
 

@Ignore @Test
public void testAllFavoredNodesDead() throws Exception {

  TableName tableName = TableName.valueOf("testAllFavoredNodesDead");
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
      HConstants.CATALOG_FAMILY));
  admin.createTable(tableDescriptor, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM);
  TEST_UTIL.waitTableAvailable(tableName);

  final RegionInfo region = admin.getRegions(tableName).get(0);
  LOG.info("Region that's supposed to be in transition: " + region);
  FavoredNodesManager fnm = master.getFavoredNodesManager();
  List<ServerName> currentFN = fnm.getFavoredNodes(region);
  assertNotNull(currentFN);

  // Lets kill all the RS that are favored nodes for this region.
  stopServersAndWaitUntilProcessed(currentFN);

  final RegionStates regionStates = master.getAssignmentManager().getRegionStates();
  TEST_UTIL.waitFor(10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return regionStates.getRegionState(region).isFailedOpen();
    }
  });

  assertTrue("Region: " + region + " should be RIT",
      regionStates.getRegionState(region).isFailedOpen());

  // Regenerate FN and assign, everything else should be fine
  List<ServerName> serversForNewFN = Lists.newArrayList();
  for (ServerName sn : admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
    .getLiveServerMetrics().keySet()) {
    serversForNewFN.add(ServerName.valueOf(sn.getHostname(), sn.getPort(), NON_STARTCODE));
  }

  FavoredNodeAssignmentHelper helper = new FavoredNodeAssignmentHelper(serversForNewFN, conf);
  helper.initialize();

  for (RegionStateNode regionState: regionStates.getRegionsInTransition()) {
    RegionInfo regionInfo = regionState.getRegionInfo();
    List<ServerName> newFavoredNodes = helper.generateFavoredNodes(regionInfo);
    assertNotNull(newFavoredNodes);
    assertEquals(FavoredNodeAssignmentHelper.FAVORED_NODES_NUM, newFavoredNodes.size());
    LOG.info("Region: " + regionInfo.getEncodedName() + " FN: " + newFavoredNodes);

    Map<RegionInfo, List<ServerName>> regionFNMap = Maps.newHashMap();
    regionFNMap.put(regionInfo, newFavoredNodes);
    fnm.updateFavoredNodes(regionFNMap);
    LOG.info("Assigning region: " + regionInfo.getEncodedName());
    admin.assign(regionInfo.getEncodedNameAsBytes());
  }
  TEST_UTIL.waitUntilNoRegionsInTransition(60000);
  assertEquals("Not all regions are online", REGION_NUM, admin.getRegions(tableName).size());

  admin.balancerSwitch(true, true);
  assertTrue("Balancer did not run", admin.balance());
  TEST_UTIL.waitUntilNoRegionsInTransition(60000);

  checkFavoredNodeAssignments(tableName, fnm, regionStates);
}
 

protected void createSchema() throws IOException {
  Configuration conf = getConf();
  TableName tableName = getTableName(conf);
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Admin admin = conn.getAdmin()) {
    if (!admin.tableExists(tableName)) {
      TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
        new TableDescriptorBuilder.ModifyableTableDescriptor(getTableName(getConf()));

      ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY_NAME);
      // if -DuseMob=true force all data through mob path.
      setMobProperties(conf, familyDescriptor);
      tableDescriptor.setColumnFamily(familyDescriptor);
      // Always add these families. Just skip writing to them when we do not test per CF flush.
      familyDescriptor =
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(BIG_FAMILY_NAME);
      setMobProperties(conf, familyDescriptor);
      tableDescriptor.setColumnFamily(familyDescriptor);

      familyDescriptor =
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TINY_FAMILY_NAME);
      setMobProperties(conf, familyDescriptor);
      tableDescriptor.setColumnFamily(familyDescriptor);

      // If we want to pre-split compute how many splits.
      if (conf.getBoolean(HBaseTestingUtility.PRESPLIT_TEST_TABLE_KEY,
          HBaseTestingUtility.PRESPLIT_TEST_TABLE)) {
        int numberOfServers = admin.getRegionServers().size();
        if (numberOfServers == 0) {
          throw new IllegalStateException("No live regionservers");
        }
        int regionsPerServer = conf.getInt(HBaseTestingUtility.REGIONS_PER_SERVER_KEY,
            HBaseTestingUtility.DEFAULT_REGIONS_PER_SERVER);
        int totalNumberOfRegions = numberOfServers * regionsPerServer;
        LOG.info("Number of live regionservers: " + numberOfServers + ", " +
            "pre-splitting table into " + totalNumberOfRegions + " regions " +
            "(default regions per server: " + regionsPerServer + ")");


        byte[][] splits = new RegionSplitter.UniformSplit().split(totalNumberOfRegions);

        admin.createTable(tableDescriptor, splits);
      } else {
        // Looks like we're just letting things play out.
        // Create a table with on region by default.
        // This will make the splitting work hard.
        admin.createTable(tableDescriptor);
      }
    }
  } catch (MasterNotRunningException e) {
    LOG.error("Master not running", e);
    throw new IOException(e);
  }
}