Java Code Examples for org.apache.hadoop.hbase.client.TableDescriptorBuilder#build()

public static TableDescriptor tableDescriptorFromThrift(TTableDescriptor in) {
  TableDescriptorBuilder builder = TableDescriptorBuilder
      .newBuilder(tableNameFromThrift(in.getTableName()));
  for (TColumnFamilyDescriptor column : in.getColumns()) {
    builder.setColumnFamily(columnFamilyDescriptorFromThrift(column));
  }
  if (in.isSetAttributes()) {
    for (Map.Entry<ByteBuffer, ByteBuffer> attribute : in.getAttributes().entrySet()) {
      builder.setValue(attribute.getKey().array(), attribute.getValue().array());
    }
  }
  if (in.isSetDurability()) {
    builder.setDurability(durabilityFromThrift(in.getDurability()));
  }
  return builder.build();
}
 

@Test
// HBASE-3516: Test CP Class loading from local file system
public void testClassLoadingFromLocalFS() throws Exception {
  File jarFile = buildCoprocessorJar(cpName3);

  // create a table that references the jar
  TableDescriptorBuilder tdb = TableDescriptorBuilder.newBuilder(TableName.valueOf(cpName3));
  tdb.setColumnFamily(ColumnFamilyDescriptorBuilder
    .newBuilder(Bytes.toBytes("test")).build());
  tdb.setValue("COPROCESSOR$1", getLocalPath(jarFile) + "|" + cpName3 + "|" +
    Coprocessor.PRIORITY_USER);
  TableDescriptor tableDescriptor = tdb.build();
  Admin admin = TEST_UTIL.getAdmin();
  admin.createTable(tableDescriptor);
  waitForTable(tableDescriptor.getTableName());

  // verify that the coprocessor was loaded
  boolean found = false;
  MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
  for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
    if (region.getRegionInfo().getRegionNameAsString().startsWith(cpName3)) {
      found = (region.getCoprocessorHost().findCoprocessor(cpName3) != null);
    }
  }
  assertTrue("Class " + cpName3 + " was missing on a region", found);
}
 

private void testRollbackAndDoubleExecution(TableDescriptorBuilder builder) throws Exception {
  // create the table
  final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
  ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, true);

  // Start the Create procedure && kill the executor
  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  builder.setRegionReplication(3);
  TableDescriptor htd = builder.build();
  RegionInfo[] regions = ModifyRegionUtils.createRegionInfos(htd, splitKeys);
  long procId = procExec.submitProcedure(
    new CreateTableProcedure(procExec.getEnvironment(), htd, regions));

  int lastStep = 2; // failing before CREATE_TABLE_WRITE_FS_LAYOUT
  MasterProcedureTestingUtility.testRollbackAndDoubleExecution(procExec, procId, lastStep);

  TableName tableName = htd.getTableName();
  MasterProcedureTestingUtility.validateTableDeletion(getMaster(), tableName);

  // are we able to create the table after a rollback?
  resetProcExecutorTestingKillFlag();
  testSimpleCreate(tableName, splitKeys);
}
 

public static void tryUpdateMetaTableDescriptor(Configuration conf, FileSystem fs, Path rootdir,
    Function<TableDescriptorBuilder, TableDescriptorBuilder> metaObserver) throws IOException {
  // see if we already have meta descriptor on fs. Write one if not.
  try {
    getTableDescriptorFromFs(fs, rootdir, TableName.META_TABLE_NAME);
  } catch (TableInfoMissingException e) {
    TableDescriptorBuilder builder = createMetaTableDescriptorBuilder(conf);
    if (metaObserver != null) {
      builder = metaObserver.apply(builder);
    }
    TableDescriptor td = builder.build();
    LOG.info("Creating new hbase:meta table descriptor {}", td);
    TableName tableName = td.getTableName();
    Path tableDir = CommonFSUtils.getTableDir(rootdir, tableName);
    Path p = writeTableDescriptor(fs, td, tableDir, getTableInfoPath(fs, tableDir, true));
    if (p == null) {
      throw new IOException("Failed update hbase:meta table descriptor");
    }
    LOG.info("Updated hbase:meta table descriptor to {}", p);
  }
}
 

private static TableDescriptor createHTableDescriptor(final int regionNum,
    final int numFamilies) {
  TableDescriptorBuilder builder =
      TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE_NAME + ":" + regionNum));
  IntStream.range(0, numFamilies)
      .mapToObj(i -> ColumnFamilyDescriptorBuilder.of(FAMILY_PREFIX + i))
      .forEachOrdered(builder::setColumnFamily);
  return builder.build();
}
 

@Test
public void testPreWALAppendNotCalledOnMetaEdit() throws Exception {
  final TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() +
      "." + name.getMethodName());
  TableDescriptorBuilder tdBuilder = TableDescriptorBuilder.newBuilder(tableName);
  ColumnFamilyDescriptorBuilder cfBuilder = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY);
  tdBuilder.setColumnFamily(cfBuilder.build());
  tdBuilder.setCoprocessor(SimpleRegionObserver.class.getName());
  TableDescriptor td = tdBuilder.build();
  Table table = util.createTable(td, new byte[][] { A, B, C });

  PreWALAppendWALActionsListener listener = new PreWALAppendWALActionsListener();
  List<HRegion> regions = util.getHBaseCluster().getRegions(tableName);
  //should be only one region
  HRegion region = regions.get(0);

  region.getWAL().registerWALActionsListener(listener);
  //flushing should write to the WAL
  region.flush(true);
  //so should compaction
  region.compact(false);
  //and so should closing the region
  region.close();

  //but we still shouldn't have triggered preWALAppend because no user data was written
  String[] methods = new String[] {"getCtPreWALAppend"};
  Object[] expectedResult = new Integer[]{0};
  verifyMethodResult(SimpleRegionObserver.class, methods, tableName, expectedResult);
}
 

public static TableDescriptor createHTD(final TableName tableName, final String... family) {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  for (int i = 0; i < family.length; ++i) {
    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(family[i]));
  }
  return builder.build();
}
 

@Before
public void setUp() throws Exception {
  conf = TEST_UTIL.getConfiguration();
  conf.setInt("hfile.format.version", 3);
  conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
  conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");

  // Create the test encryption key
  SecureRandom rng = new SecureRandom();
  byte[] keyBytes = new byte[AES.KEY_LENGTH];
  rng.nextBytes(keyBytes);
  String algorithm =
      conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
  cfKey = new SecretKeySpec(keyBytes,algorithm);

  // Start the minicluster
  TEST_UTIL.startMiniCluster(3);

  // Create the table
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf("default", "TestHBaseFsckEncryption"));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder
      .newBuilder(Bytes.toBytes("cf"))
      .setEncryptionType(algorithm)
      .setEncryptionKey(EncryptionUtil.wrapKey(conf,
        conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()),
        cfKey)).build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  tableDescriptor = tableDescriptorBuilder.build();
  TEST_UTIL.getAdmin().createTable(tableDescriptor);
  TEST_UTIL.waitTableAvailable(tableDescriptor.getTableName(), 5000);
}
 

/**
 * Create simple HTD with three families: 'a', 'b', and 'c'
 * @param tableName name of the table descriptor
 * @return
 */
private TableDescriptor createBasic3FamilyHTD(final String tableName) {
  TableDescriptorBuilder tableBuilder = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName));
  ColumnFamilyDescriptor  a = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("a"));
  tableBuilder.addColumnFamily(a);
  ColumnFamilyDescriptor b = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("b"));
  tableBuilder.addColumnFamily(b);
  ColumnFamilyDescriptor c = ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("c"));
  tableBuilder.addColumnFamily(c);
  return tableBuilder.build();
}
 

@Test
public void createTableInSystemNamespace() throws Exception {
  final TableName tableName = TableName.valueOf("hbase:" + name.getMethodName());
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(tableName);
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("cf1")).build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  admin.createTable(tableDescriptor);
  assertEquals(0, admin.listTableDescriptors().size());
  assertTrue(admin.tableExists(tableName));
  admin.disableTable(tableDescriptor.getTableName());
  admin.deleteTable(tableDescriptor.getTableName());
}
 

private RegionInfo createRandomRegion(final String name) {
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(name));
  String[] families = new String[] { "info", "anchor" };
  for (int i = 0; i < families.length; i++) {
    ColumnFamilyDescriptor columnFamilyDescriptor =
      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(families[i])).build();
    tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  }
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  return RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
}
 

/**
 * Converts a TableSchema to TableDescriptor
 * @param ts A pb TableSchema instance.
 * @return An {@link TableDescriptor} made from the passed in pb <code>ts</code>.
 */
public static TableDescriptor toTableDescriptor(final TableSchema ts) {
  TableDescriptorBuilder builder
    = TableDescriptorBuilder.newBuilder(ProtobufUtil.toTableName(ts.getTableName()));
  ts.getColumnFamiliesList()
    .stream()
    .map(ProtobufUtil::toColumnFamilyDescriptor)
    .forEach(builder::setColumnFamily);
  ts.getAttributesList()
    .forEach(a -> builder.setValue(a.getFirst().toByteArray(), a.getSecond().toByteArray()));
  ts.getConfigurationList()
    .forEach(a -> builder.setValue(a.getName(), a.getValue()));
  return builder.build();
}
 

public TableDescriptor build(TableDescriptor desc, Class<? extends IndexCodec> clazz) throws IOException {
  // add the codec for the index to the map of options
  Map<String, String> opts = this.convertToMap();
  opts.put(NonTxIndexBuilder.CODEC_CLASS_NAME_KEY, clazz.getName());
      TableDescriptorBuilder newBuilder = TableDescriptorBuilder.newBuilder(desc);
      Indexer.enableIndexing(newBuilder, NonTxIndexBuilder.class, opts, Coprocessor.PRIORITY_USER);
      return newBuilder.build();
}
 

@Before
public void setUp() throws Exception {
  UTIL = new HBaseTestingUtility();
  conf = UTIL.getConfiguration();
  conf.set(MobConstants.MOB_FILE_CACHE_SIZE_KEY, TEST_CACHE_SIZE);
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(UTIL.createTableDescriptor(
      TableName.valueOf("testMobFileCache"), ColumnFamilyDescriptorBuilder.DEFAULT_MIN_VERSIONS,
      3, HConstants.FOREVER, ColumnFamilyDescriptorBuilder.DEFAULT_KEEP_DELETED));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY1))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY2))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY3))
      .setMobEnabled(true)
      .setMobThreshold(0)
      .build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
  TableDescriptor tableDescriptor = tableDescriptorBuilder.build();
  RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  mobFileCache = new MobFileCache(conf);
  region = HBaseTestingUtility
    .createRegionAndWAL(regionInfo, UTIL.getDataTestDir(), conf, tableDescriptor, mobFileCache);
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  TestReplicationBase.setUpBeforeClass();

  connection1 = ConnectionFactory.createConnection(CONF1);
  connection2 = ConnectionFactory.createConnection(CONF2);
  admin1 = connection1.getAdmin();
  admin2 = connection2.getAdmin();

  admin1.createNamespace(NamespaceDescriptor.create(ns1).build());
  admin1.createNamespace(NamespaceDescriptor.create(ns2).build());
  admin2.createNamespace(NamespaceDescriptor.create(ns1).build());
  admin2.createNamespace(NamespaceDescriptor.create(ns2).build());

  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tabAName);
  builder.setColumnFamily(ColumnFamilyDescriptorBuilder
    .newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
  builder.setColumnFamily(ColumnFamilyDescriptorBuilder
    .newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
  TableDescriptor tabA = builder.build();
  admin1.createTable(tabA);
  admin2.createTable(tabA);

  builder = TableDescriptorBuilder.newBuilder(tabBName);
  builder.setColumnFamily(ColumnFamilyDescriptorBuilder
    .newBuilder(f1Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
  builder.setColumnFamily(ColumnFamilyDescriptorBuilder
    .newBuilder(f2Name).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build());
  TableDescriptor tabB = builder.build();
  admin1.createTable(tabB);
  admin2.createTable(tabB);
}
 

@Override
public TableDescriptor preModifyTable(ObserverContext<MasterCoprocessorEnvironment> env,
    TableName tableName, final TableDescriptor currentDescriptor,
    final TableDescriptor newDescriptor) throws IOException {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(newDescriptor);
  for (ColumnFamilyDescriptor cfd : newDescriptor.getColumnFamilies()) {
    builder.modifyColumnFamily(
        ColumnFamilyDescriptorBuilder.newBuilder(cfd).setMaxVersions(1).build());
  }
  return builder.build();
}
 

private TableDescriptor getTableDesc(TableName tableName, byte[]... families) {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  Arrays.stream(families).map(
    f -> ColumnFamilyDescriptorBuilder.newBuilder(f).setMaxVersions(Integer.MAX_VALUE).build())
      .forEachOrdered(builder::setColumnFamily);
  return builder.build();
}
 

/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    IndexTestingUtils.setupConfig(conf);
    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // check the version to see if it isn't supported
    String version = VersionInfo.getVersion();
    boolean supported = false;
    if (Indexer.validateVersion(version, conf) == null) {
        supported = true;
    }

    // start the minicluster
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    util.startMiniCluster();

    try {
        // setup the primary table
        TableDescriptorBuilder descBuilder = TableDescriptorBuilder.newBuilder(TableName.valueOf(
                "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion"));
        byte[] family = Bytes.toBytes("f");
        
        descBuilder.addColumnFamily(ColumnFamilyDescriptorBuilder.of(family));
        TableDescriptor desc=descBuilder.build();
        // enable indexing to a non-existant index table
        String indexTableName = "INDEX_TABLE";
        ColumnGroup fam1 = new ColumnGroup(indexTableName);
        fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
        CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
        builder.addIndexGroup(fam1);
        builder.build(desc);

        // get a reference to the regionserver, so we can ensure it aborts
        HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

        // create the primary table
        Admin admin = util.getAdmin();
        if (supported) {
            admin.createTable(desc);
            assertFalse("Hosting regeion server failed, even the HBase version (" + version
                    + ") supports WAL Compression.", server.isAborted());
        } else {
            admin.createTableAsync(desc, null);

            // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
            // broken.
            while (!server.isAborted()) {
                LOGGER.debug("Waiting on regionserver to abort..");
            }
        }

    } finally {
        // cleanup
        util.shutdownMiniCluster();
    }
}
 

void loadingClassFromLibDirInJar(String libPrefix) throws Exception {
  FileSystem fs = cluster.getFileSystem();

  File innerJarFile1 = buildCoprocessorJar(cpName1);
  File innerJarFile2 = buildCoprocessorJar(cpName2);
  File outerJarFile = new File(TEST_UTIL.getDataTestDir().toString(), "outer.jar");

  ClassLoaderTestHelper.addJarFilesToJar(
    outerJarFile, libPrefix, innerJarFile1, innerJarFile2);

  // copy the jars into dfs
  fs.copyFromLocalFile(new Path(outerJarFile.getPath()),
    new Path(fs.getUri().toString() + Path.SEPARATOR));
  String jarFileOnHDFS = fs.getUri().toString() + Path.SEPARATOR +
    outerJarFile.getName();
  assertTrue("Copy jar file to HDFS failed.",
    fs.exists(new Path(jarFileOnHDFS)));
  LOG.info("Copied jar file to HDFS: " + jarFileOnHDFS);

  // create a table that references the coprocessors
  TableDescriptorBuilder tdb = TableDescriptorBuilder.newBuilder(tableName);
  tdb.setColumnFamily(ColumnFamilyDescriptorBuilder
    .newBuilder(Bytes.toBytes("test")).build());
    // without configuration values
  tdb.setValue("COPROCESSOR$1", jarFileOnHDFS + "|" + cpName1
    + "|" + Coprocessor.PRIORITY_USER);
    // with configuration values
  tdb.setValue("COPROCESSOR$2", jarFileOnHDFS + "|" + cpName2
    + "|" + Coprocessor.PRIORITY_USER + "|k1=v1,k2=v2,k3=v3");
  Admin admin = TEST_UTIL.getAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }

  TableDescriptor tableDescriptor = tdb.build();
  admin.createTable(tableDescriptor);
  waitForTable(tableDescriptor.getTableName());

  // verify that the coprocessors were loaded
  boolean found1 = false, found2 = false, found2_k1 = false,
      found2_k2 = false, found2_k3 = false;
  MiniHBaseCluster hbase = TEST_UTIL.getHBaseCluster();
  for (HRegion region: hbase.getRegionServer(0).getOnlineRegionsLocalContext()) {
    if (region.getRegionInfo().getRegionNameAsString().startsWith(tableName.getNameAsString())) {
      CoprocessorEnvironment env;
      env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName1);
      if (env != null) {
        found1 = true;
      }
      env = region.getCoprocessorHost().findCoprocessorEnvironment(cpName2);
      if (env != null) {
        found2 = true;
        Configuration conf = env.getConfiguration();
        found2_k1 = conf.get("k1") != null;
        found2_k2 = conf.get("k2") != null;
        found2_k3 = conf.get("k3") != null;
      }
    }
  }
  assertTrue("Class " + cpName1 + " was missing on a region", found1);
  assertTrue("Class " + cpName2 + " was missing on a region", found2);
  assertTrue("Configuration key 'k1' was missing on a region", found2_k1);
  assertTrue("Configuration key 'k2' was missing on a region", found2_k2);
  assertTrue("Configuration key 'k3' was missing on a region", found2_k3);
}
 

public static TableDescriptor createTableDescriptor(TableName tableName, byte[]... family) {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  Stream.of(family).map(ColumnFamilyDescriptorBuilder::of)
    .forEachOrdered(builder::setColumnFamily);
  return builder.build();
}