Java Code Examples for org.apache.hadoop.hbase.HTableDescriptor#addFamily()

/**
 * Helper method which locates or creates and returns the specified HTable used for triple/ quad storage
 * @param config Hadoop Configuration of the cluster running HBase
 * @param tableName String table name
 * @param create boolean option to create the table if does not exists
 * @param splits array of keys used to pre-split new table, may be null
 * @return HTable
 * @throws IOException throws IOException in case of any HBase IO problems
 */
public static HTable getTable(Configuration config, String tableName, boolean create, byte[][] splits) throws IOException {
    Configuration cfg = HBaseConfiguration.create(config);
    cfg.setLong(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 3600000l);
    if (create) {
        try (Connection con = ConnectionFactory.createConnection(config)) {
            try (Admin admin = con.getAdmin()) {
            	    //check if the table exists and if it doesn't, make it
                if (!admin.tableExists(TableName.valueOf(tableName))) {
                    HTableDescriptor td = new HTableDescriptor(TableName.valueOf(tableName));
                    td.addFamily(createColumnFamily());
                    admin.createTable(td, splits);
                }
            }
        }
    }

    //this is deprecated, the recommendation now is to use connection.getTable()
    HTable table = new HTable(cfg, tableName);
    table.setAutoFlushTo(false);
    return table;
}
 

@BeforeClass
public static void setUpClass() throws IOException {
    connection = ConnectionFactory.createConnection();
    Admin admin = connection.getAdmin();

    for (Map.Entry<String, List<String>> tableInfo : tableToCfs.entrySet()) {
        TableName tableName = TableName.valueOf(tableInfo.getKey());
        if (!admin.tableExists(tableName)) {
            // table not found, creating
            HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
            for (String columnFamily : tableInfo.getValue()) {
                tableDescriptor.addFamily(new HColumnDescriptor(columnFamily));
            }
            admin.createTable(tableDescriptor);
        }
    }
}
 

public void init(String[] cfs) throws IOException {
    logger.debug("Checking streaming remote store for {} at {}.", tableName, String.join(", ", cfs));
    Connection conn = getConnection();
    Admin admin = conn.getAdmin();
    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(hbaseTableName));
    for (String family : cfs) {
        HColumnDescriptor fd = new HColumnDescriptor(family);
        desc.addFamily(fd);
    }
    DistributedLock lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentProcess();
    try {
        boolean locked = lock.lock(lockPath());
        if (locked && !admin.tableExists(TableName.valueOf(hbaseTableName))) {
            logger.info("Create htable with {}.", desc);
            admin.createTable(desc);
        } else {
            logger.info("Table exists or cannot fetch lock {}", desc);
        }
    } finally {
        admin.close();
        if (lock != null && lock.isLockedByMe(lockPath())) {
            lock.unlock(lockPath());
        }
    }
    table = conn.getTable(TableName.valueOf(hbaseTableName));
}
 

public static void create() throws Exception {
	HBaseAdmin admin = new HBaseAdmin(cfg);
	if (admin.tableExists(tableName)) {
		System.out.println("[info]table has created!");
	} else {
		try {
			TableName table = TableName.valueOf(tableName);
			HTableDescriptor tableDescriptor = new HTableDescriptor(table);
			tableDescriptor.addFamily(new HColumnDescriptor(familyName));

			admin.createTable(tableDescriptor);
		} catch (TableExistsException e) {
			System.out.println("[warning] table exists!");
		}
	}
	System.out.println("[info]create table success!");
}
 

public static void createTable(Connection conn, String namespace, String tableName, Algorithm compressionType) {
    // Check whether table already exists
    if (Helper.existsTable(conn, namespace, tableName)) {
        Helper.dropTable(conn, namespace, tableName);
    }
    if (!Helper.existsTable(conn, namespace, tableName)) {
        
        // Create table
        try (Admin admin = conn.getAdmin()) {
        HTableDescriptor table = new HTableDescriptor(TableName.valueOf(namespace, tableName));
        table.addFamily(new HColumnDescriptor(DATA_COLUMN_FAMILY).setCompressionType(compressionType));
        _log.debug("creating table {}", table.toString());
        admin.createTable(table);
        } 
        catch (Exception ex) {
            throw new OrcaHBaseException(ex, "Failed to create table - " + tableName);
        }
    }
}
 

/** create a HTable that has the same performance settings as normal cube table, for benchmark purpose */
public static void createBenchmarkHTable(TableName tableName, String cfName) throws IOException {
    Admin admin = HBaseConnection.get(KylinConfig.getInstanceFromEnv().getStorageUrl()).getAdmin();
    try {
        if (admin.tableExists(tableName)) {
            logger.info("disabling hbase table " + tableName);
            admin.disableTable(tableName);
            logger.info("deleting hbase table " + tableName);
            admin.deleteTable(tableName);
        }

        HTableDescriptor tableDesc = new HTableDescriptor(tableName);
        tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());

        KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
        tableDesc.addFamily(createColumnFamily(kylinConfig, cfName, false));

        logger.info("creating hbase table " + tableName);
        admin.createTable(tableDesc, null);
        Preconditions.checkArgument(admin.isTableAvailable(tableName), "table " + tableName + " created, but is not available due to some reasons");
        logger.info("create hbase table " + tableName + " done.");
    } finally {
        IOUtils.closeQuietly(admin);
    }
}
 

protected static Table createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
                                    List<String> coprocessors) throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : columnFamilies) {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family);
    columnDesc.setMaxVersions(Integer.MAX_VALUE);
    columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
    desc.addFamily(columnDesc);
  }
  if (existingData) {
    desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
  }
  // Divide individually to prevent any overflow
  int priority = Coprocessor.PRIORITY_USER;
  // order in list is the same order that coprocessors will be invoked
  for (String coprocessor : coprocessors) {
    desc.addCoprocessor(coprocessor, null, ++priority, null);
  }
  hBaseAdmin.createTable(desc);
  testUtil.waitTableAvailable(tableName, 5000);
  return testUtil.getConnection().getTable(TableName.valueOf(tableName));
}
 

public static void main(String[] args) throws Exception {
	String quorum = "192.168.0.30,192.168.0.31,192.168.0.32";
	//quorum = "192.168.8.191,192.168.1.192,192.168.1.193";
	int port = 2181;
	String znode = "/hyperbase1";
	HBaseConnPool connPool = new HBaseClientManager(quorum, port, znode);
	HBaseDDLHandler ddlHandler = new HBaseDDLHandler(connPool);

	String tableName = "demo_test";
	System.out.println("=============================== : delete");
	ddlHandler.deleteTable(tableName);

	String columnFamily = "cf";
	System.out.println("=============================== : create");
	ddlHandler.createTable(tableName, columnFamily, "cf2");

	System.out.println("=============================== : desc");
	HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
	System.out.println("=============================== : alter");
	HBaseAdmin admin = new HBaseAdmin(connPool.getConn());
	admin.disableTable(tableName);
	HTableInterface htable = ddlHandler.getTable(tableName);
	HTableDescriptor tableDesc = admin.getTableDescriptor(htable.getTableName());
	tableDesc.removeFamily(Bytes.toBytes("cf2"));
	HColumnDescriptor newhcd = new HColumnDescriptor("cf3");
	newhcd.setMaxVersions(2);
	newhcd.setKeepDeletedCells(KeepDeletedCells.TRUE);
	tableDesc.addFamily(newhcd);

	admin.modifyTable(tableName, tableDesc);
	admin.enableTable(tableName);
	admin.close();

	System.out.println("=============================== : desc");
	HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
	System.out.println("=============================== : delete");
	ddlHandler.deleteTable(tableName);

	connPool.closeConn();
}
 

@Test
public void testNamespace() throws Exception {
    TableName tn = TableName.valueOf(TEST_NAMESPACE, tableName);

    HTableDescriptor td = new HTableDescriptor(tn);
    HColumnDescriptor cd = new HColumnDescriptor(TEST_TABLE_CF);
    td.addFamily(cd);

    admin.createTable(td);

    Util.validateTable(admin, TEST_NAMESPACE + ":" + tableName);

    admin.disableTable(tn);
    admin.deleteTable(tn);
}
 

@Test
public void excessiveColumnFamilyMatch_shouldReturnTrue() {
    List<HTableDescriptor> expectedSchemas = Arrays.asList(
            createHtd("table1", "table1_1"),
            createHtd("table2", "table2_1", "table2_2", "table2_3"),
            createHtd("table3"));
    List<HTableDescriptor> actualSchemas = copySchema(expectedSchemas);
    for (HTableDescriptor htd : actualSchemas) {
        htd.addFamily(new HColumnDescriptor("newCF"));
    }
    assertThat(verifier.verifySchemas(expectedSchemas, actualSchemas), is(true));
}
 

/**
 * {@inheritDoc}
 */
@Override
public HTableDescriptor getTableDescriptor() throws IOException {
    HTableDescriptor table = new HTableDescriptor(tableName);
    for (String columnFamily : columnFamilies) {
        table.addFamily(new HColumnDescriptor(columnFamily));
    }
    return table;
}
 

private static void createHTableIfNeeded(Connection conn, String tableName) throws IOException {
    Admin hbase = conn.getAdmin();

    try {
        boolean tableExist = false;
        try {
            hbase.getTableDescriptor(TableName.valueOf(tableName));
            tableExist = true;
        } catch (TableNotFoundException e) {
            //do nothing?
        }

        if (tableExist) {
            logger.info("HTable '{}' already exists", tableName);
            return;
        }

        logger.info("Creating HTable '{}'", tableName);

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));

        HColumnDescriptor fd = new HColumnDescriptor(CF);
        fd.setBlocksize(CELL_SIZE);
        desc.addFamily(fd);
        hbase.createTable(desc);

        logger.info("HTable '{}' created", tableName);
    } finally {
        hbase.close();
    }
}
 

/**
   * Create the global transaction table with the given configuration.
   * 
   * @param conf
   * @throws IOException
   */
  public static void createTable(final Configuration conf) throws IOException {
      HTableDescriptor tableDesc = new HTableDescriptor(TABLE_NAME);
      tableDesc.addFamily(new HColumnDescriptor(INFO_FAMILY));
      @SuppressWarnings("resource")
HBaseAdmin admin = new HBaseAdmin(conf);
      if (!admin.tableExists(TABLE_NAME)) {
          admin.createTable(tableDesc);
      }
  }
 

/**
 * Create simple HTD with three families: 'a', 'b', and 'c'
 * @param tableName name of the table descriptor
 * @return
 */
private HTableDescriptor createBasic3FamilyHTD(final String tableName) {
  @SuppressWarnings("deprecation")
  HTableDescriptor htd = new HTableDescriptor(tableName);
  HColumnDescriptor a = new HColumnDescriptor(Bytes.toBytes("a"));
  htd.addFamily(a);
  HColumnDescriptor b = new HColumnDescriptor(Bytes.toBytes("b"));
  htd.addFamily(b);
  HColumnDescriptor c = new HColumnDescriptor(Bytes.toBytes("c"));
  htd.addFamily(c);
  return htd;
}
 

/**
 * @param args
 */
public static void main(String[] args) {
  Configuration conf = HBaseConfiguration.create();
  

  byte[] columnFamily = Bytes.toBytes("f");

  String tableName = "t";

  try {
    ZKUtil.applyClusterKeyToConf(conf, "edh1:2181:/hbase");
    HBaseAdmin hba = new HBaseAdmin(conf);
    if (hba.tableExists(tableName)) {
      hba.disableTable(tableName);
      hba.deleteTable(tableName);
    }
    HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
    HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamily);
    columnDescriptor.setMaxVersions(1);
    columnDescriptor.setBloomFilterType(BloomType.ROW);
    tableDescriptor.addFamily(columnDescriptor);
    hba.createTable(tableDescriptor);
    hba.close();
  } catch (IOException e) {
    e.printStackTrace();
  }

}
 

public static void createSchema(Configuration hbaseConf) throws IOException {
    Admin admin = ConnectionFactory.createConnection(hbaseConf).getAdmin();
    if (!admin.tableExists(TableName.valueOf("sep-user-demo"))) {
        HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf("sep-user-demo"));

        HColumnDescriptor infoCf = new HColumnDescriptor("info");
        infoCf.setScope(1);
        tableDescriptor.addFamily(infoCf);

        admin.createTable(tableDescriptor);
    }
    admin.close();
}
 

private void createTestTable() throws IOException {
    HBaseAdmin admin = hBaseUtils.getHBaseAdmin();
    HTableDescriptor test_table_desc = new HTableDescriptor(TableName.valueOf(TEST_TABLE));
    HColumnDescriptor datafam = new HColumnDescriptor(TEST_FAMILY);
    HColumnDescriptor datafam2 = new HColumnDescriptor(TEST_FAMILY2);
    datafam.setMaxVersions(Integer.MAX_VALUE);
    datafam2.setMaxVersions(Integer.MAX_VALUE);
    test_table_desc.addFamily(datafam);
    test_table_desc.addFamily(datafam2);
    admin.createTable(test_table_desc);
}
 

@Override
    public void connect(Graph graph, Callback<Boolean> callback) {
        if (isConnected) {
            if (callback != null) {
                callback.on(null);
            }
            return;
        }
        this.graph = graph;

        // Create a connection to the cluster.
        try {
            Configuration conf = HBaseConfiguration.create();
            conf.set("hbase.zookeeper.quorum", "192.168.25.60");
//            conf.set("hbase.zookeeper.quorum", "localhost");
            conf.set("hbase.zookeeper.property.clientPort", "2280");

//            conf.set("zookeeper.znode.parent", "/hbase");


//            conf.set("hbase.master", "192.168.25.60:16000");

            connection = ConnectionFactory.createConnection(conf);
            HBaseAdmin.checkHBaseAvailable(conf);

            Admin admin = connection.getAdmin();


            if (!admin.tableExists(TableName.valueOf(tablename))) {
                HTableDescriptor tableDescriptor = new HTableDescriptor(TableName.valueOf(tablename));
                tableDescriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY));
                admin.createTable(tableDescriptor);
            }
            table = connection.getTable(TableName.valueOf(tablename));

            isConnected = true;
            if (callback != null) {
                callback.on(true);
            }

        } catch (Exception e) {
            e.printStackTrace();
            if (callback != null) {
                callback.on(null);
            }
        }


    }
 

public static void createHTable(CubeSegment cubeSegment, byte[][] splitKeys) throws IOException {
    String tableName = cubeSegment.getStorageLocationIdentifier();
    CubeInstance cubeInstance = cubeSegment.getCubeInstance();
    CubeDesc cubeDesc = cubeInstance.getDescriptor();
    KylinConfig kylinConfig = cubeDesc.getConfig();

    HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(cubeSegment.getStorageLocationIdentifier()));
    tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
    tableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
    tableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));

    if (!StringUtils.isEmpty(kylinConfig.getKylinOwner())) {
        //HTableOwner is the team that provides kylin service
        tableDesc.setValue(IRealizationConstants.HTableOwner, kylinConfig.getKylinOwner());
    }

    String commitInfo = KylinVersion.getGitCommitInfo();
    if (!StringUtils.isEmpty(commitInfo)) {
        tableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
    }

    //HTableUser is the cube owner, which will be the "user"
    tableDesc.setValue(IRealizationConstants.HTableUser, cubeInstance.getOwner());

    tableDesc.setValue(IRealizationConstants.HTableSegmentTag, cubeSegment.toString());

    Configuration conf = HBaseConnection.getCurrentHBaseConfiguration();
    Connection conn = HBaseConnection.get(kylinConfig.getStorageUrl());
    Admin admin = conn.getAdmin();

    try {
        if (User.isHBaseSecurityEnabled(conf)) {
            // add coprocessor for bulk load
            tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
        }

        for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHbaseMapping().getColumnFamily()) {
            HColumnDescriptor cf = createColumnFamily(kylinConfig, cfDesc.getName(), cfDesc.isMemoryHungry());
            tableDesc.addFamily(cf);
        }

        if (admin.tableExists(TableName.valueOf(tableName))) {
            // admin.disableTable(tableName);
            // admin.deleteTable(tableName);
            throw new RuntimeException("HBase table " + tableName + " exists!");
        }

        DeployCoprocessorCLI.deployCoprocessor(tableDesc);

        admin.createTable(tableDesc, splitKeys);
        Preconditions.checkArgument(admin.isTableAvailable(TableName.valueOf(tableName)), "table " + tableName + " created, but is not available due to some reasons");
        logger.info("create hbase table " + tableName + " done.");
    } finally {
        IOUtils.closeQuietly(admin);
    }

}
 

@Override
public void addColumnFamilyToTableDescriptor(HTableDescriptor tdesc, HColumnDescriptor cdesc)
{
    tdesc.addFamily(cdesc);
}