Java Code Examples for org.apache.hadoop.hbase.HTableDescriptor

The following examples show how to use org.apache.hadoop.hbase.HTableDescriptor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: phoenix   Source File: ConnectionQueryServicesImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public HTableDescriptor getTableDescriptor(byte[] tableName) throws SQLException {
    HTableInterface htable = getTable(tableName);
    try {
        return htable.getTableDescriptor();
    } catch (IOException e) {
        if(e instanceof org.apache.hadoop.hbase.TableNotFoundException ||
            e.getCause() instanceof org.apache.hadoop.hbase.TableNotFoundException) {
          byte[][] schemaAndTableName = new byte[2][];
          SchemaUtil.getVarChars(tableName, schemaAndTableName);
          throw new TableNotFoundException(Bytes.toString(schemaAndTableName[0]), Bytes.toString(schemaAndTableName[1]));
        }
        throw new RuntimeException(e);
    } finally {
        Closeables.closeQuietly(htable);
    }
}
 
Example 2
/**
 * Create the prune state table given the {@link TableName} if the table doesn't exist already.
 *
 * @param stateTable prune state table name
 */
protected void createPruneTable(TableName stateTable) throws IOException {
  try (Admin admin = this.connection.getAdmin()) {
    if (admin.tableExists(stateTable)) {
      LOG.debug("Not creating pruneStateTable {} since it already exists.",
                stateTable.getNameWithNamespaceInclAsString());
      return;
    }

    HTableDescriptor htd = new HTableDescriptor(stateTable);
    htd.addFamily(new HColumnDescriptor(DataJanitorState.FAMILY).setMaxVersions(1));
    admin.createTable(htd);
    LOG.info("Created pruneTable {}", stateTable.getNameWithNamespaceInclAsString());
  } catch (TableExistsException ex) {
    // Expected if the prune state table is being created at the same time by another client
    LOG.debug("Not creating pruneStateTable {} since it already exists.",
              stateTable.getNameWithNamespaceInclAsString(), ex);
  }
}
 
Example 3
Source Project: phoenix-tephra   Source File: AbstractHBaseTableTest.java    License: Apache License 2.0 6 votes vote down vote up
protected static Table createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
                                    List<String> coprocessors) throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : columnFamilies) {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family);
    columnDesc.setMaxVersions(Integer.MAX_VALUE);
    columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
    desc.addFamily(columnDesc);
  }
  if (existingData) {
    desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
  }
  // Divide individually to prevent any overflow
  int priority = Coprocessor.PRIORITY_USER;
  // order in list is the same order that coprocessors will be invoked
  for (String coprocessor : coprocessors) {
    desc.addCoprocessor(coprocessor, null, ++priority, null);
  }
  hBaseAdmin.createTable(desc);
  testUtil.waitTableAvailable(tableName, 5000);
  return testUtil.getConnection().getTable(TableName.valueOf(tableName));
}
 
Example 4
Source Project: incubator-atlas   Source File: HBaseBasedAuditRepository.java    License: Apache License 2.0 6 votes vote down vote up
private void createTableIfNotExists() throws AtlasException {
    Admin admin = null;
    try {
        admin = connection.getAdmin();
        LOG.info("Checking if table {} exists", tableName.getNameAsString());
        if (!admin.tableExists(tableName)) {
            LOG.info("Creating table {}", tableName.getNameAsString());
            HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
            HColumnDescriptor columnFamily = new HColumnDescriptor(COLUMN_FAMILY);
            columnFamily.setMaxVersions(1);
            columnFamily.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
            columnFamily.setCompressionType(Compression.Algorithm.GZ);
            columnFamily.setBloomFilterType(BloomType.ROW);
            tableDescriptor.addFamily(columnFamily);
            admin.createTable(tableDescriptor);
        } else {
            LOG.info("Table {} exists", tableName.getNameAsString());
        }
    } catch (IOException e) {
        throw new AtlasException(e);
    } finally {
        close(admin);
    }
}
 
Example 5
Source Project: kylin-on-parquet-v2   Source File: HFileOutputFormat3.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Serialize column family to block size map to configuration.
 * Invoked while configuring the MR job for incremental load.
 * @param tableDescriptor to read the properties from
 * @param conf to persist serialized values into
 *
 * @throws IOException
 *           on failure to read column family descriptors
 */
@VisibleForTesting
static void configureBlockSize(HTableDescriptor tableDescriptor, Configuration conf)
        throws UnsupportedEncodingException {
    StringBuilder blockSizeConfigValue = new StringBuilder();
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;
    }
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            blockSizeConfigValue.append('&');
        }
        blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        blockSizeConfigValue.append('=');
        blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8"));
    }
    // Get rid of the last ampersand
    conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString());
}
 
Example 6
Source Project: DistributedCrawler   Source File: HBaseCleaner.java    License: Apache License 2.0 6 votes vote down vote up
public static void clean(String tableName) throws IOException{
	HBaseAdmin admin = new HBaseAdmin(HBasePool.getInstance().getConf());
	admin.disableTable(tableName);
	admin.deleteTable(tableName);
	/*for(HTableDescriptor d : admin.listTables()){
		System.out.println(d.getNameAsString());
		for(HColumnDescriptor cd : d.getColumnFamilies()){
			System.out.println("===="+cd.getNameAsString());
		}
	}*/
	HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);  
       tableDescriptor.addFamily(new HColumnDescriptor("content"));  
       tableDescriptor.addFamily(new HColumnDescriptor("title"));  
	admin.createTable(tableDescriptor);
	admin.close();
}
 
Example 7
Source Project: java-docs-samples   Source File: HelloWorldTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeClass() {
  projectId = requireEnv("GOOGLE_CLOUD_PROJECT");
  instanceId = requireEnv(INSTANCE_ENV);
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(TABLE_ID));
    descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY_NAME));
    admin.createTable(descriptor);

    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));

    String rowKey = "phone#4c410523#20190401";
    Put put = new Put(Bytes.toBytes(rowKey));

    put.addColumn(
        Bytes.toBytes(COLUMN_FAMILY_NAME), Bytes.toBytes("os_name"), Bytes.toBytes("android"));
    table.put(put);

  } catch (Exception e) {
    System.out.println("Error during beforeClass: \n" + e.toString());
  }
}
 
Example 8
Source Project: kylin-on-parquet-v2   Source File: RemoteDictionaryStore.java    License: Apache License 2.0 6 votes vote down vote up
public void init(String[] cfs) throws IOException {
    logger.debug("Checking streaming remote store for {} at {}.", tableName, String.join(", ", cfs));
    Connection conn = getConnection();
    Admin admin = conn.getAdmin();
    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(hbaseTableName));
    for (String family : cfs) {
        HColumnDescriptor fd = new HColumnDescriptor(family);
        desc.addFamily(fd);
    }
    DistributedLock lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentProcess();
    try {
        boolean locked = lock.lock(lockPath());
        if (locked && !admin.tableExists(TableName.valueOf(hbaseTableName))) {
            logger.info("Create htable with {}.", desc);
            admin.createTable(desc);
        } else {
            logger.info("Table exists or cannot fetch lock {}", desc);
        }
    } finally {
        admin.close();
        if (lock != null && lock.isLockedByMe(lockPath())) {
            lock.unlock(lockPath());
        }
    }
    table = conn.getTable(TableName.valueOf(hbaseTableName));
}
 
Example 9
Source Project: hbase   Source File: TestConstraints.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testSimpleReadWrite() throws Throwable {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
  Constraints.add(desc, WorksConstraint.class);

  List<? extends Constraint> constraints = Constraints.getConstraints(desc,
      this.getClass().getClassLoader());
  assertEquals(1, constraints.size());

  assertEquals(WorksConstraint.class, constraints.get(0).getClass());

  // Check that we can add more than 1 constraint and that ordering is
  // preserved
  Constraints.add(desc, AlsoWorks.class, NameConstraint.class);
  constraints = Constraints.getConstraints(desc, this.getClass()
      .getClassLoader());
  assertEquals(3, constraints.size());

  assertEquals(WorksConstraint.class, constraints.get(0).getClass());
  assertEquals(AlsoWorks.class, constraints.get(1).getClass());
  assertEquals(NameConstraint.class, constraints.get(2).getClass());

}
 
Example 10
Source Project: canal-1.1.3   Source File: HbaseTemplate.java    License: Apache License 2.0 6 votes vote down vote up
public void createTable(String tableName, String... familyNames) {
    try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) {

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
        // 添加列簇
        if (familyNames != null) {
            for (String familyName : familyNames) {
                HColumnDescriptor hcd = new HColumnDescriptor(familyName);
                desc.addFamily(hcd);
            }
        }
        admin.createTable(desc);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example 11
Source Project: examples   Source File: Describe.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
  // Instantiate default HBase configuration object.
  // Configuration file must be in the classpath
  Configuration conf = HBaseConfiguration.create();
  // tag::DESCRIBE
  HBaseAdmin admin = new HBaseAdmin(conf);
  HTableDescriptor desc = admin.getTableDescriptor(TableName.valueOf("crc"));
  Collection<HColumnDescriptor> families = desc.getFamilies();
  System.out.println("Table " + desc.getTableName() + " has " + families.size() + " family(ies)");
  for (Iterator<HColumnDescriptor> iterator = families.iterator(); iterator.hasNext();) {
    HColumnDescriptor family = iterator.next();
    System.out.println("Family details: " + family);
  }
  // end::DESCRIBE
  admin.close();
}
 
Example 12
Source Project: kylin   Source File: DstClusterUtil.java    License: Apache License 2.0 6 votes vote down vote up
public boolean checkExist(TableName htableName, CubeSegment segment) throws IOException {
    if (!htableExists(htableName)) {
        return false;
    }
    Table table = hbaseConn.getTable(htableName);
    HTableDescriptor tableDesc = table.getTableDescriptor();
    if (segment.toString().equals(tableDesc.getValue(HTableSegmentTag))) {
        if (hbaseAdmin.isTableEnabled(htableName)) {
            return true;
        } else {
            hbaseAdmin.deleteTable(htableName);
            logger.info("htable {} is deleted", htableName);
            return false;
        }
    }
    throw new RuntimeException(
            "htable name " + htableName + " has been used by " + tableDesc.getValue(HTableSegmentTag));
}
 
Example 13
Source Project: phoenix-omid   Source File: TestHBaseTimestampStorage.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeMethod
public void setUp() throws Exception {
    HBaseAdmin admin = testutil.getHBaseAdmin();

    if (!admin.tableExists(TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME))) {
        HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
        HColumnDescriptor datafam = new HColumnDescriptor(DEFAULT_TIMESTAMP_STORAGE_CF_NAME);
        datafam.setMaxVersions(Integer.MAX_VALUE);
        desc.addFamily(datafam);

        admin.createTable(desc);
    }

    if (admin.isTableDisabled(TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME))) {
        admin.enableTable(TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME));
    }
    HTableDescriptor[] tables = admin.listTables();
    for (HTableDescriptor t : tables) {
        LOG.info(t.getNameAsString());
    }
}
 
Example 14
Source Project: hbase   Source File: VisibilityController.java    License: Apache License 2.0 6 votes vote down vote up
/********************************* Master related hooks **********************************/

  @Override
  public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
    // Need to create the new system table for labels here
    if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getConnection(), LABELS_TABLE_NAME)) {
      TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
        new TableDescriptorBuilder.ModifyableTableDescriptor(LABELS_TABLE_NAME);
      ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(LABELS_TABLE_FAMILY);
      familyDescriptor.setBloomFilterType(BloomType.NONE);
      // We will cache all the labels. No need of normal
      // table block cache.
      familyDescriptor.setBlockCacheEnabled(false);
      tableDescriptor.setColumnFamily(familyDescriptor);
      // Let the "labels" table having only one region always. We are not expecting too many labels in
      // the system.
      tableDescriptor.setValue(HTableDescriptor.SPLIT_POLICY,
          DisabledRegionSplitPolicy.class.getName());
      try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) {
        admin.createTable(tableDescriptor);
      }
    }
  }
 
Example 15
Source Project: phoenix   Source File: IndexLoadBalancerIT.java    License: Apache License 2.0 6 votes vote down vote up
private void createUserAndIndexTable(TableName tableName, TableName indexTableName)
        throws IOException {
    HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(new HColumnDescriptor("cf"));
    char c = 'A';
    byte[][] split = new byte[20][];
    for (int i = 0; i < 20; i++) {
        byte[] b = { (byte) c };
        split[i] = b;
        c++;
    }
    admin.createTable(htd, split);
    HTableDescriptor iHtd = new HTableDescriptor(indexTableName);
    iHtd.addFamily(new HColumnDescriptor("cf"));
    iHtd.setValue(IndexLoadBalancer.PARENT_TABLE_KEY, tableName.toBytes());
    admin.createTable(iHtd, split);
}
 
Example 16
protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (Admin admin = connection.getAdmin()) {
    HTableDescriptor[] tableDescriptors = admin.listTables();
    LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
    if (tableDescriptors != null) {
      for (HTableDescriptor tableDescriptor : tableDescriptors) {
        if (isTransactionalTable(tableDescriptor)) {
          List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
          LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
          if (tableRegions != null) {
            for (HRegionInfo region : tableRegions) {
              regions.add(region.getRegionName());
            }
          }
        } else {
          LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
        }
      }
    }
  }
  return regions;
}
 
Example 17
Source Project: phoenix-tephra   Source File: AbstractHBaseTableTest.java    License: Apache License 2.0 6 votes vote down vote up
protected static Table createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
                                    List<String> coprocessors) throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : columnFamilies) {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family);
    columnDesc.setMaxVersions(Integer.MAX_VALUE);
    columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
    desc.addFamily(columnDesc);
  }
  if (existingData) {
    desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
  }
  // Divide individually to prevent any overflow
  int priority = Coprocessor.PRIORITY_USER;
  // order in list is the same order that coprocessors will be invoked
  for (String coprocessor : coprocessors) {
    desc.addCoprocessor(coprocessor, null, ++priority, null);
  }
  hBaseAdmin.createTable(desc);
  testUtil.waitTableAvailable(tableName, 5000);
  return testUtil.getConnection().getTable(TableName.valueOf(tableName));
}
 
Example 18
@Test
public void testAlterColumnFamilyProperty() throws Exception {

    Properties props = new Properties(TEST_PROPERTIES);
    Connection conn = DriverManager.getConnection(getUrl(), props);
    
    String ddl = "CREATE TABLE test_table " +
            "  (a_string varchar not null, col1 integer" +
            "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
    try {
            conn.createStatement().execute(ddl);
          
            conn.createStatement().execute("ALTER TABLE TEST_TABLE ADD col2 integer IN_MEMORY=true");
            
            HTableInterface htable1 = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("TEST_TABLE")); 
            HTableDescriptor htableDesciptor1 = htable1.getTableDescriptor();
            HColumnDescriptor hcolumnDescriptor1 = htableDesciptor1.getFamily(Bytes.toBytes("_0"));
            assertTrue(hcolumnDescriptor1.isInMemory());
           
            try {
                
                conn.createStatement().execute("ALTER TABLE TEST_TABLE SET IN_MEMORY=false");
                fail("Should have caught exception.");
                
            } catch (SQLException e) {
                assertTrue(e.getMessage(), e.getMessage().contains("ERROR 1025 (42Y84): Unsupported property set in ALTER TABLE command."));
            } 
    }finally {
        conn.close();
    }
 }
 
Example 19
/**
 * Test a table modification adding a coprocessor path
 * which is whitelisted. The coprocessor should be added to
 * the table descriptor successfully.
 * @param whitelistedPaths A String array of paths to add in
 *         for the whitelisting configuration
 * @param coprocessorPath A String to use as the
 *         path for a mock coprocessor
 */
private static void negativeTestCase(String[] whitelistedPaths,
    String coprocessorPath) throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.setInt("hbase.client.retries.number", 5);
  // load coprocessor under test
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      CoprocessorWhitelistMasterObserver.class.getName());
  // set retries low to raise exception quickly
  // set a coprocessor whitelist path for test
  conf.setStrings(
      CoprocessorWhitelistMasterObserver.CP_COPROCESSOR_WHITELIST_PATHS_KEY,
      whitelistedPaths);
  UTIL.startMiniCluster();
  UTIL.createTable(TEST_TABLE, new byte[][] { TEST_FAMILY });
  UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
  Connection connection = ConnectionFactory.createConnection(conf);
  Admin admin = connection.getAdmin();
  // disable table so we do not actually try loading non-existant
  // coprocessor file
  admin.disableTable(TEST_TABLE);
  Table t = connection.getTable(TEST_TABLE);
  HTableDescriptor htd = new HTableDescriptor(t.getDescriptor());
  htd.addCoprocessor("net.clayb.hbase.coprocessor.Whitelisted",
    new Path(coprocessorPath),
    Coprocessor.PRIORITY_USER, null);
  LOG.info("Modifying Table");
  admin.modifyTable(htd);
  assertEquals(1, t.getDescriptor().getCoprocessorDescriptors().size());
  LOG.info("Done Modifying Table");
}
 
Example 20
@Override
public void initialize() throws IOException{
    try(HBaseAdmin hBaseAdmin=testUtility.getHBaseAdmin()){
        HTableDescriptor table = generateDefaultSIGovernedTable("1440");
        if (hBaseAdmin.tableExists(table.getTableName())) {
            hBaseAdmin.disableTable(table.getTableName());
            hBaseAdmin.deleteTable(table.getTableName());
        }
        hBaseAdmin.createTable(table);
    }
}
 
Example 21
Source Project: phoenix   Source File: IndexMasterObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
        TableName tableName, HTableDescriptor htd) throws IOException {
    HTableDescriptor oldDesc =
            ctx.getEnvironment().getMasterServices().getTableDescriptors().get(tableName);
    if (oldDesc.getValue(IndexLoadBalancer.PARENT_TABLE_KEY) == null
            && htd.getValue(IndexLoadBalancer.PARENT_TABLE_KEY) != null) {
        TableName userTableName =
                TableName.valueOf(htd.getValue(IndexLoadBalancer.PARENT_TABLE_KEY));
        balancer.addTablesToColocate(userTableName, htd.getTableName());
    }
    super.preModifyTableHandler(ctx, tableName, htd);
}
 
Example 22
Source Project: phoenix-tephra   Source File: TransactionProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void start(CoprocessorEnvironment e) throws IOException {
  if (e instanceof RegionCoprocessorEnvironment) {
    RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
    this.cacheSupplier = getTransactionStateCacheSupplier(env);
    this.cache = cacheSupplier.get();

    HTableDescriptor tableDesc = env.getRegion().getTableDesc();
    for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
      String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
      long ttl = 0;
      if (columnTTL != null) {
        try {
          ttl = Long.parseLong(columnTTL);
          LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
        } catch (NumberFormatException nfe) {
          LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
                     ", value = " + columnTTL);
        }
      }
      ttlByFamily.put(columnDesc.getName(), ttl);
    }

    this.allowEmptyValues = getAllowEmptyValues(env, tableDesc);
    this.txMaxLifetimeMillis = getTxMaxLifetimeMillis(env);
    this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
    if (readNonTxnData) {
      LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
    }
    initializePruneState(env);
  }
}
 
Example 23
Source Project: Kylin   Source File: ExportHBaseData.java    License: Apache License 2.0 5 votes vote down vote up
public void exportTables() throws IOException {
    cli.execute("mkdir -p " + exportFolder);

    for (HTableDescriptor table : allTables) {
        String tName = table.getNameAsString();
        if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))
            continue;

        cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportFolder + tName);
    }

    cli.execute("hadoop fs -copyToLocal " + exportFolder + " " + exportFolder);
    cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportFolder + " .");
    downloadToLocal();
}
 
Example 24
Source Project: hbase   Source File: TestReplicasClient.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  // enable store file refreshing
  HTU.getConfiguration().setInt(
      StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);
  HTU.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
  HTU.getConfiguration().setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true);
  StartMiniClusterOption option = StartMiniClusterOption.builder().numRegionServers(1).
      numAlwaysStandByMasters(1).numMasters(1).build();
  HTU.startMiniCluster(option);

  // Create table then get the single region for our new table.
  HTableDescriptor hdt = HTU.createTableDescriptor(
    TableName.valueOf(TestReplicasClient.class.getSimpleName()),
    HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
    HColumnDescriptor.DEFAULT_KEEP_DELETED);
  hdt.addCoprocessor(SlowMeCopro.class.getName());
  HTU.createTable(hdt, new byte[][]{f}, null);
  TABLE_NAME = hdt.getTableName();
  try (RegionLocator locator = HTU.getConnection().getRegionLocator(hdt.getTableName())) {
    hriPrimary = locator.getRegionLocation(row, false).getRegion();
  }

  // mock a secondary region info to open
  hriSecondary = RegionReplicaUtil.getRegionInfoForReplica(hriPrimary, 1);

  // No master
  LOG.info("Master is going to be stopped");
  TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU);
  Configuration c = new Configuration(HTU.getConfiguration());
  c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
  LOG.info("Master has stopped");
}
 
Example 25
public static void main(String[] args) throws IOException {
	if (args.length == 0) {
		System.out.println("CreateTable {tableName} {columnFamilyName}");
		return;
	}

	String tableName = args[0];
	String columnFamilyName = args[1];

	HBaseAdmin admin = new HBaseAdmin(new Configuration());

	HTableDescriptor tableDescriptor = new HTableDescriptor(); 
	tableDescriptor.setName(Bytes.toBytes(tableName));

	HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamilyName);

	columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY);
	columnDescriptor.setBlocksize(64 * 1024);
	columnDescriptor.setBloomFilterType(BloomType.ROW);

	tableDescriptor.addFamily(columnDescriptor);

	//tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());

	System.out.println("-Creating Table");
	admin.createTable(tableDescriptor);

	admin.close();
	System.out.println("-Done");
}
 
Example 26
public void testRegionReplicaReplication(int regionReplication) throws Exception {
  // test region replica replication. Create a table with single region, write some data
  // ensure that data is replicated to the secondary region
  TableName tableName = TableName.valueOf("testRegionReplicaReplicationWithReplicas_"
      + regionReplication);
  HTableDescriptor htd = HTU.createTableDescriptor(TableName.valueOf(tableName.toString()),
    HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
    HColumnDescriptor.DEFAULT_KEEP_DELETED);
  htd.setRegionReplication(regionReplication);
  HTU.getAdmin().createTable(htd);
  TableName tableNameNoReplicas =
      TableName.valueOf("testRegionReplicaReplicationWithReplicas_NO_REPLICAS");
  HTU.deleteTableIfAny(tableNameNoReplicas);
  HTU.createTable(tableNameNoReplicas, HBaseTestingUtility.fam1);

  Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
  Table table = connection.getTable(tableName);
  Table tableNoReplicas = connection.getTable(tableNameNoReplicas);

  try {
    // load some data to the non-replicated table
    HTU.loadNumericRows(tableNoReplicas, HBaseTestingUtility.fam1, 6000, 7000);

    // load the data to the table
    HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000);

    verifyReplication(tableName, regionReplication, 0, 1000);

  } finally {
    table.close();
    tableNoReplicas.close();
    HTU.deleteTableIfAny(tableNameNoReplicas);
    connection.close();
  }
}
 
Example 27
/** Add an index to a table. */
public void addIndex(final byte[] baseTableName, final IndexSpecification indexSpec) throws IOException {
    LOG.info("Adding index [" + indexSpec.getIndexId() + "] to existing table [" + Bytes.toString(baseTableName)
            + "], this may take a long time");
    // TODO, make table read-only
    LOG.warn("Not putting table in readonly, if its being written to, the index may get out of sync");
    HTableDescriptor indexTableDesc = createIndexTableDesc(baseTableName, indexSpec);
    super.createTable(indexTableDesc);
    super.disableTable(baseTableName);
    IndexedTableDescriptor indexDesc = new IndexedTableDescriptor(super.getTableDescriptor(baseTableName));
    indexDesc.addIndex(indexSpec);
    super.modifyTable(baseTableName, indexDesc.getBaseTableDescriptor());
    super.enableTable(baseTableName);
    reIndexTable(baseTableName, indexSpec);
}
 
Example 28
Source Project: hbase   Source File: AbstractTestWALReplay.java    License: Apache License 2.0 5 votes vote down vote up
private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(Bytes.toBytes("a"));
  tableDescriptor.setColumnFamily(familyDescriptor);
  return new HTableDescriptor(tableDescriptor);
}
 
Example 29
Source Project: kylin-on-parquet-v2   Source File: LookupTableToHFileJob.java    License: Apache License 2.0 5 votes vote down vote up
/**
 *
 * @param sourceTableName
 * @param sourceTable
 * @param kylinConfig
 * @return Pair of HTableName and shard number
 * @throws IOException
 */
private Pair<String, Integer> createHTable(String sourceTableName, IReadableTable sourceTable,
        KylinConfig kylinConfig) throws IOException {
    TableSignature signature = sourceTable.getSignature();
    int shardNum = calculateShardNum(kylinConfig, signature.getSize());
    Connection conn = getHBaseConnection(kylinConfig);
    Admin admin = conn.getAdmin();
    String hTableName = genHTableName(kylinConfig, admin, sourceTableName);

    TableName tableName = TableName.valueOf(hTableName);
    HTableDescriptor hTableDesc = new HTableDescriptor(tableName);
    hTableDesc.setCompactionEnabled(false);
    hTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
    hTableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
    hTableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
    String commitInfo = KylinVersion.getGitCommitInfo();
    if (!StringUtils.isEmpty(commitInfo)) {
        hTableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
    }

    HColumnDescriptor cf = CubeHTableUtil.createColumnFamily(kylinConfig, HBaseLookupRowEncoder.CF_STRING, false);
    hTableDesc.addFamily(cf);

    try {
        if (shardNum > 1) {
            admin.createTable(hTableDesc, getSplitsByShardNum(shardNum));
        } else {
            admin.createTable(hTableDesc);
        }
    } finally {
        IOUtils.closeQuietly(admin);
    }
    return new Pair<>(hTableName, shardNum);
}
 
Example 30
Source Project: hbase   Source File: TestConstraints.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that Constraints are properly enabled, disabled, and removed
 *
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test
public void testEnableDisableRemove() throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
  // check general enabling/disabling of constraints
  // first add a constraint
  Constraints.add(desc, AllPassConstraint.class);
  // make sure everything is enabled
  assertTrue(Constraints.enabled(desc, AllPassConstraint.class));
  assertTrue(desc.hasCoprocessor(ConstraintProcessor.class.getName()));

  // check disabling
  Constraints.disable(desc);
  assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName()));
  // make sure the added constraints are still present
  assertTrue(Constraints.enabled(desc, AllPassConstraint.class));

  // check just removing the single constraint
  Constraints.remove(desc, AllPassConstraint.class);
  assertFalse(Constraints.has(desc, AllPassConstraint.class));

  // Add back the single constraint
  Constraints.add(desc, AllPassConstraint.class);

  // and now check that when we remove constraints, all are gone
  Constraints.remove(desc);
  assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName()));
  assertFalse(Constraints.has(desc, AllPassConstraint.class));

}