org.apache.hadoop.hbase.HTableDescriptor Java Examples

The following examples show how to use org.apache.hadoop.hbase.HTableDescriptor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConnectionQueryServicesImpl.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public HTableDescriptor getTableDescriptor(byte[] tableName) throws SQLException {
    HTableInterface htable = getTable(tableName);
    try {
        return htable.getTableDescriptor();
    } catch (IOException e) {
        if(e instanceof org.apache.hadoop.hbase.TableNotFoundException ||
            e.getCause() instanceof org.apache.hadoop.hbase.TableNotFoundException) {
          byte[][] schemaAndTableName = new byte[2][];
          SchemaUtil.getVarChars(tableName, schemaAndTableName);
          throw new TableNotFoundException(Bytes.toString(schemaAndTableName[0]), Bytes.toString(schemaAndTableName[1]));
        }
        throw new RuntimeException(e);
    } finally {
        Closeables.closeQuietly(htable);
    }
}
 
Example #2
Source File: HBaseTransactionPruningPlugin.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
/**
 * Create the prune state table given the {@link TableName} if the table doesn't exist already.
 *
 * @param stateTable prune state table name
 */
protected void createPruneTable(TableName stateTable) throws IOException {
  try (Admin admin = this.connection.getAdmin()) {
    if (admin.tableExists(stateTable)) {
      LOG.debug("Not creating pruneStateTable {} since it already exists.",
                stateTable.getNameWithNamespaceInclAsString());
      return;
    }

    HTableDescriptor htd = new HTableDescriptor(stateTable);
    htd.addFamily(new HColumnDescriptor(DataJanitorState.FAMILY).setMaxVersions(1));
    admin.createTable(htd);
    LOG.info("Created pruneTable {}", stateTable.getNameWithNamespaceInclAsString());
  } catch (TableExistsException ex) {
    // Expected if the prune state table is being created at the same time by another client
    LOG.debug("Not creating pruneStateTable {} since it already exists.",
              stateTable.getNameWithNamespaceInclAsString(), ex);
  }
}
 
Example #3
Source File: AbstractHBaseTableTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
protected static Table createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
                                    List<String> coprocessors) throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : columnFamilies) {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family);
    columnDesc.setMaxVersions(Integer.MAX_VALUE);
    columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
    desc.addFamily(columnDesc);
  }
  if (existingData) {
    desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
  }
  // Divide individually to prevent any overflow
  int priority = Coprocessor.PRIORITY_USER;
  // order in list is the same order that coprocessors will be invoked
  for (String coprocessor : coprocessors) {
    desc.addCoprocessor(coprocessor, null, ++priority, null);
  }
  hBaseAdmin.createTable(desc);
  testUtil.waitTableAvailable(tableName, 5000);
  return testUtil.getConnection().getTable(TableName.valueOf(tableName));
}
 
Example #4
Source File: HBaseBasedAuditRepository.java    From incubator-atlas with Apache License 2.0 6 votes vote down vote up
private void createTableIfNotExists() throws AtlasException {
    Admin admin = null;
    try {
        admin = connection.getAdmin();
        LOG.info("Checking if table {} exists", tableName.getNameAsString());
        if (!admin.tableExists(tableName)) {
            LOG.info("Creating table {}", tableName.getNameAsString());
            HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
            HColumnDescriptor columnFamily = new HColumnDescriptor(COLUMN_FAMILY);
            columnFamily.setMaxVersions(1);
            columnFamily.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
            columnFamily.setCompressionType(Compression.Algorithm.GZ);
            columnFamily.setBloomFilterType(BloomType.ROW);
            tableDescriptor.addFamily(columnFamily);
            admin.createTable(tableDescriptor);
        } else {
            LOG.info("Table {} exists", tableName.getNameAsString());
        }
    } catch (IOException e) {
        throw new AtlasException(e);
    } finally {
        close(admin);
    }
}
 
Example #5
Source File: HFileOutputFormat3.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
/**
 * Serialize column family to block size map to configuration.
 * Invoked while configuring the MR job for incremental load.
 * @param tableDescriptor to read the properties from
 * @param conf to persist serialized values into
 *
 * @throws IOException
 *           on failure to read column family descriptors
 */
@VisibleForTesting
static void configureBlockSize(HTableDescriptor tableDescriptor, Configuration conf)
        throws UnsupportedEncodingException {
    StringBuilder blockSizeConfigValue = new StringBuilder();
    if (tableDescriptor == null) {
        // could happen with mock table instance
        return;
    }
    Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
    int i = 0;
    for (HColumnDescriptor familyDescriptor : families) {
        if (i++ > 0) {
            blockSizeConfigValue.append('&');
        }
        blockSizeConfigValue.append(URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
        blockSizeConfigValue.append('=');
        blockSizeConfigValue.append(URLEncoder.encode(String.valueOf(familyDescriptor.getBlocksize()), "UTF-8"));
    }
    // Get rid of the last ampersand
    conf.set(BLOCK_SIZE_FAMILIES_CONF_KEY, blockSizeConfigValue.toString());
}
 
Example #6
Source File: HBaseCleaner.java    From DistributedCrawler with Apache License 2.0 6 votes vote down vote up
public static void clean(String tableName) throws IOException{
	HBaseAdmin admin = new HBaseAdmin(HBasePool.getInstance().getConf());
	admin.disableTable(tableName);
	admin.deleteTable(tableName);
	/*for(HTableDescriptor d : admin.listTables()){
		System.out.println(d.getNameAsString());
		for(HColumnDescriptor cd : d.getColumnFamilies()){
			System.out.println("===="+cd.getNameAsString());
		}
	}*/
	HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);  
       tableDescriptor.addFamily(new HColumnDescriptor("content"));  
       tableDescriptor.addFamily(new HColumnDescriptor("title"));  
	admin.createTable(tableDescriptor);
	admin.close();
}
 
Example #7
Source File: HelloWorldTest.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeClass() {
  projectId = requireEnv("GOOGLE_CLOUD_PROJECT");
  instanceId = requireEnv(INSTANCE_ENV);
  try (Connection connection = BigtableConfiguration.connect(projectId, instanceId)) {
    Admin admin = connection.getAdmin();
    HTableDescriptor descriptor = new HTableDescriptor(TableName.valueOf(TABLE_ID));
    descriptor.addFamily(new HColumnDescriptor(COLUMN_FAMILY_NAME));
    admin.createTable(descriptor);

    Table table = connection.getTable(TableName.valueOf(Bytes.toBytes(TABLE_ID)));

    String rowKey = "phone#4c410523#20190401";
    Put put = new Put(Bytes.toBytes(rowKey));

    put.addColumn(
        Bytes.toBytes(COLUMN_FAMILY_NAME), Bytes.toBytes("os_name"), Bytes.toBytes("android"));
    table.put(put);

  } catch (Exception e) {
    System.out.println("Error during beforeClass: \n" + e.toString());
  }
}
 
Example #8
Source File: RemoteDictionaryStore.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
public void init(String[] cfs) throws IOException {
    logger.debug("Checking streaming remote store for {} at {}.", tableName, String.join(", ", cfs));
    Connection conn = getConnection();
    Admin admin = conn.getAdmin();
    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(hbaseTableName));
    for (String family : cfs) {
        HColumnDescriptor fd = new HColumnDescriptor(family);
        desc.addFamily(fd);
    }
    DistributedLock lock = KylinConfig.getInstanceFromEnv().getDistributedLockFactory().lockForCurrentProcess();
    try {
        boolean locked = lock.lock(lockPath());
        if (locked && !admin.tableExists(TableName.valueOf(hbaseTableName))) {
            logger.info("Create htable with {}.", desc);
            admin.createTable(desc);
        } else {
            logger.info("Table exists or cannot fetch lock {}", desc);
        }
    } finally {
        admin.close();
        if (lock != null && lock.isLockedByMe(lockPath())) {
            lock.unlock(lockPath());
        }
    }
    table = conn.getTable(TableName.valueOf(hbaseTableName));
}
 
Example #9
Source File: TestConstraints.java    From hbase with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testSimpleReadWrite() throws Throwable {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
  Constraints.add(desc, WorksConstraint.class);

  List<? extends Constraint> constraints = Constraints.getConstraints(desc,
      this.getClass().getClassLoader());
  assertEquals(1, constraints.size());

  assertEquals(WorksConstraint.class, constraints.get(0).getClass());

  // Check that we can add more than 1 constraint and that ordering is
  // preserved
  Constraints.add(desc, AlsoWorks.class, NameConstraint.class);
  constraints = Constraints.getConstraints(desc, this.getClass()
      .getClassLoader());
  assertEquals(3, constraints.size());

  assertEquals(WorksConstraint.class, constraints.get(0).getClass());
  assertEquals(AlsoWorks.class, constraints.get(1).getClass());
  assertEquals(NameConstraint.class, constraints.get(2).getClass());

}
 
Example #10
Source File: HbaseTemplate.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
public void createTable(String tableName, String... familyNames) {
    try (HBaseAdmin admin = (HBaseAdmin) getConnection().getAdmin()) {

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
        // 添加列簇
        if (familyNames != null) {
            for (String familyName : familyNames) {
                HColumnDescriptor hcd = new HColumnDescriptor(familyName);
                desc.addFamily(hcd);
            }
        }
        admin.createTable(desc);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example #11
Source File: Describe.java    From examples with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws MasterNotRunningException, ZooKeeperConnectionException, IOException {
  // Instantiate default HBase configuration object.
  // Configuration file must be in the classpath
  Configuration conf = HBaseConfiguration.create();
  // tag::DESCRIBE
  HBaseAdmin admin = new HBaseAdmin(conf);
  HTableDescriptor desc = admin.getTableDescriptor(TableName.valueOf("crc"));
  Collection<HColumnDescriptor> families = desc.getFamilies();
  System.out.println("Table " + desc.getTableName() + " has " + families.size() + " family(ies)");
  for (Iterator<HColumnDescriptor> iterator = families.iterator(); iterator.hasNext();) {
    HColumnDescriptor family = iterator.next();
    System.out.println("Family details: " + family);
  }
  // end::DESCRIBE
  admin.close();
}
 
Example #12
Source File: DstClusterUtil.java    From kylin with Apache License 2.0 6 votes vote down vote up
public boolean checkExist(TableName htableName, CubeSegment segment) throws IOException {
    if (!htableExists(htableName)) {
        return false;
    }
    Table table = hbaseConn.getTable(htableName);
    HTableDescriptor tableDesc = table.getTableDescriptor();
    if (segment.toString().equals(tableDesc.getValue(HTableSegmentTag))) {
        if (hbaseAdmin.isTableEnabled(htableName)) {
            return true;
        } else {
            hbaseAdmin.deleteTable(htableName);
            logger.info("htable {} is deleted", htableName);
            return false;
        }
    }
    throw new RuntimeException(
            "htable name " + htableName + " has been used by " + tableDesc.getValue(HTableSegmentTag));
}
 
Example #13
Source File: TestHBaseTimestampStorage.java    From phoenix-omid with Apache License 2.0 6 votes vote down vote up
@BeforeMethod
public void setUp() throws Exception {
    HBaseAdmin admin = testutil.getHBaseAdmin();

    if (!admin.tableExists(TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME))) {
        HTableDescriptor desc = new HTableDescriptor(TABLE_NAME);
        HColumnDescriptor datafam = new HColumnDescriptor(DEFAULT_TIMESTAMP_STORAGE_CF_NAME);
        datafam.setMaxVersions(Integer.MAX_VALUE);
        desc.addFamily(datafam);

        admin.createTable(desc);
    }

    if (admin.isTableDisabled(TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME))) {
        admin.enableTable(TableName.valueOf(DEFAULT_TIMESTAMP_STORAGE_TABLE_NAME));
    }
    HTableDescriptor[] tables = admin.listTables();
    for (HTableDescriptor t : tables) {
        LOG.info(t.getNameAsString());
    }
}
 
Example #14
Source File: VisibilityController.java    From hbase with Apache License 2.0 6 votes vote down vote up
/********************************* Master related hooks **********************************/

  @Override
  public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
    // Need to create the new system table for labels here
    if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getConnection(), LABELS_TABLE_NAME)) {
      TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
        new TableDescriptorBuilder.ModifyableTableDescriptor(LABELS_TABLE_NAME);
      ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(LABELS_TABLE_FAMILY);
      familyDescriptor.setBloomFilterType(BloomType.NONE);
      // We will cache all the labels. No need of normal
      // table block cache.
      familyDescriptor.setBlockCacheEnabled(false);
      tableDescriptor.setColumnFamily(familyDescriptor);
      // Let the "labels" table having only one region always. We are not expecting too many labels in
      // the system.
      tableDescriptor.setValue(HTableDescriptor.SPLIT_POLICY,
          DisabledRegionSplitPolicy.class.getName());
      try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) {
        admin.createTable(tableDescriptor);
      }
    }
  }
 
Example #15
Source File: IndexLoadBalancerIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private void createUserAndIndexTable(TableName tableName, TableName indexTableName)
        throws IOException {
    HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(new HColumnDescriptor("cf"));
    char c = 'A';
    byte[][] split = new byte[20][];
    for (int i = 0; i < 20; i++) {
        byte[] b = { (byte) c };
        split[i] = b;
        c++;
    }
    admin.createTable(htd, split);
    HTableDescriptor iHtd = new HTableDescriptor(indexTableName);
    iHtd.addFamily(new HColumnDescriptor("cf"));
    iHtd.setValue(IndexLoadBalancer.PARENT_TABLE_KEY, tableName.toBytes());
    admin.createTable(iHtd, split);
}
 
Example #16
Source File: HBaseTransactionPruningPlugin.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
protected SortedSet<byte[]> getTransactionalRegions() throws IOException {
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  try (Admin admin = connection.getAdmin()) {
    HTableDescriptor[] tableDescriptors = admin.listTables();
    LOG.debug("Got {} tables to process", tableDescriptors == null ? 0 : tableDescriptors.length);
    if (tableDescriptors != null) {
      for (HTableDescriptor tableDescriptor : tableDescriptors) {
        if (isTransactionalTable(tableDescriptor)) {
          List<HRegionInfo> tableRegions = admin.getTableRegions(tableDescriptor.getTableName());
          LOG.debug("Regions for table {}: {}", tableDescriptor.getTableName(), tableRegions);
          if (tableRegions != null) {
            for (HRegionInfo region : tableRegions) {
              regions.add(region.getRegionName());
            }
          }
        } else {
          LOG.debug("{} is not a transactional table", tableDescriptor.getTableName());
        }
      }
    }
  }
  return regions;
}
 
Example #17
Source File: AbstractHBaseTableTest.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
protected static Table createTable(byte[] tableName, byte[][] columnFamilies, boolean existingData,
                                    List<String> coprocessors) throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
  for (byte[] family : columnFamilies) {
    HColumnDescriptor columnDesc = new HColumnDescriptor(family);
    columnDesc.setMaxVersions(Integer.MAX_VALUE);
    columnDesc.setValue(TxConstants.PROPERTY_TTL, String.valueOf(100000)); // in millis
    desc.addFamily(columnDesc);
  }
  if (existingData) {
    desc.setValue(TxConstants.READ_NON_TX_DATA, "true");
  }
  // Divide individually to prevent any overflow
  int priority = Coprocessor.PRIORITY_USER;
  // order in list is the same order that coprocessors will be invoked
  for (String coprocessor : coprocessors) {
    desc.addCoprocessor(coprocessor, null, ++priority, null);
  }
  hBaseAdmin.createTable(desc);
  testUtil.waitTableAvailable(tableName, 5000);
  return testUtil.getConnection().getTable(TableName.valueOf(tableName));
}
 
Example #18
Source File: AlterTableTest.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Test
public void testAlterColumnFamilyProperty() throws Exception {

    Properties props = new Properties(TEST_PROPERTIES);
    Connection conn = DriverManager.getConnection(getUrl(), props);
    
    String ddl = "CREATE TABLE test_table " +
            "  (a_string varchar not null, col1 integer" +
            "  CONSTRAINT pk PRIMARY KEY (a_string))\n";
    try {
            conn.createStatement().execute(ddl);
          
            conn.createStatement().execute("ALTER TABLE TEST_TABLE ADD col2 integer IN_MEMORY=true");
            
            HTableInterface htable1 = conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes("TEST_TABLE")); 
            HTableDescriptor htableDesciptor1 = htable1.getTableDescriptor();
            HColumnDescriptor hcolumnDescriptor1 = htableDesciptor1.getFamily(Bytes.toBytes("_0"));
            assertTrue(hcolumnDescriptor1.isInMemory());
           
            try {
                
                conn.createStatement().execute("ALTER TABLE TEST_TABLE SET IN_MEMORY=false");
                fail("Should have caught exception.");
                
            } catch (SQLException e) {
                assertTrue(e.getMessage(), e.getMessage().contains("ERROR 1025 (42Y84): Unsupported property set in ALTER TABLE command."));
            } 
    }finally {
        conn.close();
    }
 }
 
Example #19
Source File: TestCoprocessorWhitelistMasterObserver.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test a table modification adding a coprocessor path
 * which is whitelisted. The coprocessor should be added to
 * the table descriptor successfully.
 * @param whitelistedPaths A String array of paths to add in
 *         for the whitelisting configuration
 * @param coprocessorPath A String to use as the
 *         path for a mock coprocessor
 */
private static void negativeTestCase(String[] whitelistedPaths,
    String coprocessorPath) throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.setInt("hbase.client.retries.number", 5);
  // load coprocessor under test
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      CoprocessorWhitelistMasterObserver.class.getName());
  // set retries low to raise exception quickly
  // set a coprocessor whitelist path for test
  conf.setStrings(
      CoprocessorWhitelistMasterObserver.CP_COPROCESSOR_WHITELIST_PATHS_KEY,
      whitelistedPaths);
  UTIL.startMiniCluster();
  UTIL.createTable(TEST_TABLE, new byte[][] { TEST_FAMILY });
  UTIL.waitUntilAllRegionsAssigned(TEST_TABLE);
  Connection connection = ConnectionFactory.createConnection(conf);
  Admin admin = connection.getAdmin();
  // disable table so we do not actually try loading non-existant
  // coprocessor file
  admin.disableTable(TEST_TABLE);
  Table t = connection.getTable(TEST_TABLE);
  HTableDescriptor htd = new HTableDescriptor(t.getDescriptor());
  htd.addCoprocessor("net.clayb.hbase.coprocessor.Whitelisted",
    new Path(coprocessorPath),
    Coprocessor.PRIORITY_USER, null);
  LOG.info("Modifying Table");
  admin.modifyTable(htd);
  assertEquals(1, t.getDescriptor().getCoprocessorDescriptors().size());
  LOG.info("Done Modifying Table");
}
 
Example #20
Source File: HBaseSITestEnv.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void initialize() throws IOException{
    try(HBaseAdmin hBaseAdmin=testUtility.getHBaseAdmin()){
        HTableDescriptor table = generateDefaultSIGovernedTable("1440");
        if (hBaseAdmin.tableExists(table.getTableName())) {
            hBaseAdmin.disableTable(table.getTableName());
            hBaseAdmin.deleteTable(table.getTableName());
        }
        hBaseAdmin.createTable(table);
    }
}
 
Example #21
Source File: IndexMasterObserver.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void preModifyTableHandler(ObserverContext<MasterCoprocessorEnvironment> ctx,
        TableName tableName, HTableDescriptor htd) throws IOException {
    HTableDescriptor oldDesc =
            ctx.getEnvironment().getMasterServices().getTableDescriptors().get(tableName);
    if (oldDesc.getValue(IndexLoadBalancer.PARENT_TABLE_KEY) == null
            && htd.getValue(IndexLoadBalancer.PARENT_TABLE_KEY) != null) {
        TableName userTableName =
                TableName.valueOf(htd.getValue(IndexLoadBalancer.PARENT_TABLE_KEY));
        balancer.addTablesToColocate(userTableName, htd.getTableName());
    }
    super.preModifyTableHandler(ctx, tableName, htd);
}
 
Example #22
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public void start(CoprocessorEnvironment e) throws IOException {
  if (e instanceof RegionCoprocessorEnvironment) {
    RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
    this.cacheSupplier = getTransactionStateCacheSupplier(env);
    this.cache = cacheSupplier.get();

    HTableDescriptor tableDesc = env.getRegion().getTableDesc();
    for (HColumnDescriptor columnDesc : tableDesc.getFamilies()) {
      String columnTTL = columnDesc.getValue(TxConstants.PROPERTY_TTL);
      long ttl = 0;
      if (columnTTL != null) {
        try {
          ttl = Long.parseLong(columnTTL);
          LOG.info("Family " + columnDesc.getNameAsString() + " has TTL of " + columnTTL);
        } catch (NumberFormatException nfe) {
          LOG.warn("Invalid TTL value configured for column family " + columnDesc.getNameAsString() +
                     ", value = " + columnTTL);
        }
      }
      ttlByFamily.put(columnDesc.getName(), ttl);
    }

    this.allowEmptyValues = getAllowEmptyValues(env, tableDesc);
    this.txMaxLifetimeMillis = getTxMaxLifetimeMillis(env);
    this.readNonTxnData = Boolean.valueOf(tableDesc.getValue(TxConstants.READ_NON_TX_DATA));
    if (readNonTxnData) {
      LOG.info("Reading pre-existing data enabled for table " + tableDesc.getNameAsString());
    }
    initializePruneState(env);
  }
}
 
Example #23
Source File: ExportHBaseData.java    From Kylin with Apache License 2.0 5 votes vote down vote up
public void exportTables() throws IOException {
    cli.execute("mkdir -p " + exportFolder);

    for (HTableDescriptor table : allTables) {
        String tName = table.getNameAsString();
        if (!tName.equals(tableNameBase) && !tName.startsWith(HBaseMiniclusterHelper.SHARED_STORAGE_PREFIX))
            continue;

        cli.execute("hbase org.apache.hadoop.hbase.mapreduce.Export " + tName + " " + exportFolder + tName);
    }

    cli.execute("hadoop fs -copyToLocal " + exportFolder + " " + exportFolder);
    cli.execute("tar -zcvf " + backupArchive + " --directory=" + exportFolder + " .");
    downloadToLocal();
}
 
Example #24
Source File: TestReplicasClient.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  // enable store file refreshing
  HTU.getConfiguration().setInt(
      StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, REFRESH_PERIOD);
  HTU.getConfiguration().setBoolean("hbase.client.log.scanner.activity", true);
  HTU.getConfiguration().setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true);
  StartMiniClusterOption option = StartMiniClusterOption.builder().numRegionServers(1).
      numAlwaysStandByMasters(1).numMasters(1).build();
  HTU.startMiniCluster(option);

  // Create table then get the single region for our new table.
  HTableDescriptor hdt = HTU.createTableDescriptor(
    TableName.valueOf(TestReplicasClient.class.getSimpleName()),
    HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
    HColumnDescriptor.DEFAULT_KEEP_DELETED);
  hdt.addCoprocessor(SlowMeCopro.class.getName());
  HTU.createTable(hdt, new byte[][]{f}, null);
  TABLE_NAME = hdt.getTableName();
  try (RegionLocator locator = HTU.getConnection().getRegionLocator(hdt.getTableName())) {
    hriPrimary = locator.getRegionLocation(row, false).getRegion();
  }

  // mock a secondary region info to open
  hriSecondary = RegionReplicaUtil.getRegionInfoForReplica(hriPrimary, 1);

  // No master
  LOG.info("Master is going to be stopped");
  TestRegionServerNoMaster.stopMasterAndAssignMeta(HTU);
  Configuration c = new Configuration(HTU.getConfiguration());
  c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
  LOG.info("Master has stopped");
}
 
Example #25
Source File: HBaseCreateTable.java    From Kafka-Spark-Hbase-Example with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws IOException {
	if (args.length == 0) {
		System.out.println("CreateTable {tableName} {columnFamilyName}");
		return;
	}

	String tableName = args[0];
	String columnFamilyName = args[1];

	HBaseAdmin admin = new HBaseAdmin(new Configuration());

	HTableDescriptor tableDescriptor = new HTableDescriptor(); 
	tableDescriptor.setName(Bytes.toBytes(tableName));

	HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamilyName);

	columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY);
	columnDescriptor.setBlocksize(64 * 1024);
	columnDescriptor.setBloomFilterType(BloomType.ROW);

	tableDescriptor.addFamily(columnDescriptor);

	//tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());

	System.out.println("-Creating Table");
	admin.createTable(tableDescriptor);

	admin.close();
	System.out.println("-Done");
}
 
Example #26
Source File: TestRegionReplicaReplicationEndpoint.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void testRegionReplicaReplication(int regionReplication) throws Exception {
  // test region replica replication. Create a table with single region, write some data
  // ensure that data is replicated to the secondary region
  TableName tableName = TableName.valueOf("testRegionReplicaReplicationWithReplicas_"
      + regionReplication);
  HTableDescriptor htd = HTU.createTableDescriptor(TableName.valueOf(tableName.toString()),
    HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
    HColumnDescriptor.DEFAULT_KEEP_DELETED);
  htd.setRegionReplication(regionReplication);
  HTU.getAdmin().createTable(htd);
  TableName tableNameNoReplicas =
      TableName.valueOf("testRegionReplicaReplicationWithReplicas_NO_REPLICAS");
  HTU.deleteTableIfAny(tableNameNoReplicas);
  HTU.createTable(tableNameNoReplicas, HBaseTestingUtility.fam1);

  Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
  Table table = connection.getTable(tableName);
  Table tableNoReplicas = connection.getTable(tableNameNoReplicas);

  try {
    // load some data to the non-replicated table
    HTU.loadNumericRows(tableNoReplicas, HBaseTestingUtility.fam1, 6000, 7000);

    // load the data to the table
    HTU.loadNumericRows(table, HBaseTestingUtility.fam1, 0, 1000);

    verifyReplication(tableName, regionReplication, 0, 1000);

  } finally {
    table.close();
    tableNoReplicas.close();
    HTU.deleteTableIfAny(tableNameNoReplicas);
    connection.close();
  }
}
 
Example #27
Source File: IndexedTableAdmin.java    From hbase-secondary-index with GNU General Public License v3.0 5 votes vote down vote up
/** Add an index to a table. */
public void addIndex(final byte[] baseTableName, final IndexSpecification indexSpec) throws IOException {
    LOG.info("Adding index [" + indexSpec.getIndexId() + "] to existing table [" + Bytes.toString(baseTableName)
            + "], this may take a long time");
    // TODO, make table read-only
    LOG.warn("Not putting table in readonly, if its being written to, the index may get out of sync");
    HTableDescriptor indexTableDesc = createIndexTableDesc(baseTableName, indexSpec);
    super.createTable(indexTableDesc);
    super.disableTable(baseTableName);
    IndexedTableDescriptor indexDesc = new IndexedTableDescriptor(super.getTableDescriptor(baseTableName));
    indexDesc.addIndex(indexSpec);
    super.modifyTable(baseTableName, indexDesc.getBaseTableDescriptor());
    super.enableTable(baseTableName);
    reIndexTable(baseTableName, indexSpec);
}
 
Example #28
Source File: AbstractTestWALReplay.java    From hbase with Apache License 2.0 5 votes vote down vote up
private HTableDescriptor createBasic1FamilyHTD(final TableName tableName) {
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(Bytes.toBytes("a"));
  tableDescriptor.setColumnFamily(familyDescriptor);
  return new HTableDescriptor(tableDescriptor);
}
 
Example #29
Source File: LookupTableToHFileJob.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
/**
 *
 * @param sourceTableName
 * @param sourceTable
 * @param kylinConfig
 * @return Pair of HTableName and shard number
 * @throws IOException
 */
private Pair<String, Integer> createHTable(String sourceTableName, IReadableTable sourceTable,
        KylinConfig kylinConfig) throws IOException {
    TableSignature signature = sourceTable.getSignature();
    int shardNum = calculateShardNum(kylinConfig, signature.getSize());
    Connection conn = getHBaseConnection(kylinConfig);
    Admin admin = conn.getAdmin();
    String hTableName = genHTableName(kylinConfig, admin, sourceTableName);

    TableName tableName = TableName.valueOf(hTableName);
    HTableDescriptor hTableDesc = new HTableDescriptor(tableName);
    hTableDesc.setCompactionEnabled(false);
    hTableDesc.setValue(HTableDescriptor.SPLIT_POLICY, DisabledRegionSplitPolicy.class.getName());
    hTableDesc.setValue(IRealizationConstants.HTableTag, kylinConfig.getMetadataUrlPrefix());
    hTableDesc.setValue(IRealizationConstants.HTableCreationTime, String.valueOf(System.currentTimeMillis()));
    String commitInfo = KylinVersion.getGitCommitInfo();
    if (!StringUtils.isEmpty(commitInfo)) {
        hTableDesc.setValue(IRealizationConstants.HTableGitTag, commitInfo);
    }

    HColumnDescriptor cf = CubeHTableUtil.createColumnFamily(kylinConfig, HBaseLookupRowEncoder.CF_STRING, false);
    hTableDesc.addFamily(cf);

    try {
        if (shardNum > 1) {
            admin.createTable(hTableDesc, getSplitsByShardNum(shardNum));
        } else {
            admin.createTable(hTableDesc);
        }
    } finally {
        IOUtils.closeQuietly(admin);
    }
    return new Pair<>(hTableName, shardNum);
}
 
Example #30
Source File: TestConstraints.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test that Constraints are properly enabled, disabled, and removed
 *
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test
public void testEnableDisableRemove() throws Exception {
  HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
  // check general enabling/disabling of constraints
  // first add a constraint
  Constraints.add(desc, AllPassConstraint.class);
  // make sure everything is enabled
  assertTrue(Constraints.enabled(desc, AllPassConstraint.class));
  assertTrue(desc.hasCoprocessor(ConstraintProcessor.class.getName()));

  // check disabling
  Constraints.disable(desc);
  assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName()));
  // make sure the added constraints are still present
  assertTrue(Constraints.enabled(desc, AllPassConstraint.class));

  // check just removing the single constraint
  Constraints.remove(desc, AllPassConstraint.class);
  assertFalse(Constraints.has(desc, AllPassConstraint.class));

  // Add back the single constraint
  Constraints.add(desc, AllPassConstraint.class);

  // and now check that when we remove constraints, all are gone
  Constraints.remove(desc);
  assertFalse(desc.hasCoprocessor(ConstraintProcessor.class.getName()));
  assertFalse(Constraints.has(desc, AllPassConstraint.class));

}