org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy Java Examples

The following examples show how to use org.apache.hadoop.hbase.regionserver.ConstantSizeRegionSplitPolicy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSnapshotMetadata.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) {
  // enable snapshot support
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around
  // some files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  conf.setInt("hbase.regionserver.msginterval", 100);
  conf.setBoolean("hbase.master.enabletable.roundrobin", true);
  // Avoid potentially aggressive splitting which would cause snapshot to fail
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());
}
 
Example #2
Source File: TestSnapshotTemporaryDirectory.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(TEMP_DIR, ".tmpDir").toUri());
}
 
Example #3
Source File: TestSnapshotDFSTemporaryDirectory.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) throws IOException {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());

  String snapshotPath = UTIL.getDefaultRootDirPath().toString() + Path.SEPARATOR +
      UUID.randomUUID().toString() + Path.SEPARATOR + ".tmpdir" + Path.SEPARATOR;
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(snapshotPath).toUri());
}
 
Example #4
Source File: TestSnapshotCloneIndependence.java    From hbase with Apache License 2.0 6 votes vote down vote up
static void setupConf(Configuration conf) {
  // Up the handlers; this test needs more than usual.
  conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 15);
  // enable snapshot support
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around
  // some files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  conf.setInt("hbase.regionserver.msginterval", 100);
  conf.setBoolean("hbase.master.enabletable.roundrobin", true);
  // Avoid potentially aggressive splitting which would cause snapshot to fail
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());
  // Execute cleaner frequently to induce failures
  conf.setInt("hbase.master.cleaner.interval", CLEANER_INTERVAL);
  conf.setInt("hbase.master.hfilecleaner.plugins.snapshot.period", CLEANER_INTERVAL);
  // Effectively disable TimeToLiveHFileCleaner. Don't want to fully disable it because that
  // will even trigger races between creating the directory containing back references and
  // the back reference itself.
  conf.setInt("hbase.master.hfilecleaner.ttl", CLEANER_INTERVAL);
}
 
Example #5
Source File: TestSnapshotFromClient.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());

}
 
Example #6
Source File: TestSnapshotFromMaster.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 2);
  conf.setInt("hbase.hstore.compactionThreshold", 5);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", blockingStoreFiles);
  // Ensure no extra cleaners on by default (e.g. TimeToLiveHFileCleaner)
  conf.set(HFileCleaner.MASTER_HFILE_CLEANER_PLUGINS, "");
  conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, "");
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.setLong(SnapshotManager.HBASE_SNAPSHOT_SENTINELS_CLEANUP_TIMEOUT_MILLIS, 3 * 1000L);
  conf.setLong(SnapshotHFileCleaner.HFILE_CACHE_REFRESH_PERIOD_CONF_KEY, cacheRefreshPeriod);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());
  conf.setInt("hbase.hfile.compactions.cleaner.interval", 20 * 1000);
  conf.setInt("hbase.master.cleaner.snapshot.interval", 500);
}
 
Example #7
Source File: TestRegionSnapshotTask.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Table setupTable(TableName tableName) throws Exception {
  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  // Flush many files, but do not compact immediately
  // Make sure that the region does not split
  builder
      .setMemStoreFlushSize(5000)
      .setRegionSplitPolicyClassName(ConstantSizeRegionSplitPolicy.class.getName())
      .setMaxFileSize(100 * 1024 * 1024)
      .setValue("hbase.hstore.compactionThreshold", "250");

  TableDescriptor td = builder.build();
  byte[] fam = Bytes.toBytes("fam");
  Table table = TEST_UTIL.createTable(td, new byte[][] {fam},
      TEST_UTIL.getConfiguration());
  TEST_UTIL.loadTable(table, fam);
  return table;
}
 
Example #8
Source File: TestFlushSnapshotFromClient.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());
}
 
Example #9
Source File: TestHFileArchiving.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // drop the memstore size so we get flushes
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // disable major compactions
  conf.setInt(HConstants.MAJOR_COMPACTION_PERIOD, 0);

  // prevent aggressive region split
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());
}
 
Example #10
Source File: AcidGuaranteesTestBase.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Set small flush size for minicluster so we exercise reseeking scanners
  Configuration conf = UTIL.getConfiguration();
  conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024));
  // prevent aggressive region split
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());
  conf.setInt("hfile.format.version", 3); // for mob tests
  UTIL.startMiniCluster(1);
}
 
Example #11
Source File: TestFuzzyRowFilterEndToEnd.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @throws java.lang.Exception
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setInt("hbase.client.scanner.caching", 1000);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());
  // set no splits
  conf.setLong(HConstants.HREGION_MAX_FILESIZE, (1024L) * 1024 * 1024 * 10);

  TEST_UTIL.startMiniCluster();
}
 
Example #12
Source File: IntegrationTestAcidGuarantees.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void setUpCluster() throws Exception {
  // Set small flush size for minicluster so we exercise reseeking scanners
  util = getTestingUtil(getConf());
  util.initializeCluster(SERVER_COUNT);
  conf = getConf();
  conf.set(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, String.valueOf(128 * 1024));
  // prevent aggressive region split
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
    ConstantSizeRegionSplitPolicy.class.getName());

  tool = new AcidGuaranteesTestTool();
  tool.setConf(getConf());
}
 
Example #13
Source File: ChangeSplitPolicyAction.java    From hbase with Apache License 2.0 5 votes vote down vote up
public ChangeSplitPolicyAction(TableName tableName) {
  this.tableName = tableName;
  possiblePolicies = new String[] {
      IncreasingToUpperBoundRegionSplitPolicy.class.getName(),
      ConstantSizeRegionSplitPolicy.class.getName(),
      DisabledRegionSplitPolicy.class.getName()
  };
  this.random = new Random();
}
 
Example #14
Source File: App.java    From hadoop-arch-book with Apache License 2.0 5 votes vote down vote up
private static boolean createTable(byte[] tableName, byte[] columnFamilyName,
    short regionCount, long regionMaxSize, HBaseAdmin admin)
    throws IOException {

  if (admin.tableExists(tableName)) {
    return false;
  }

  HTableDescriptor tableDescriptor = new HTableDescriptor();
  tableDescriptor.setName(tableName);

  HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamilyName);

  columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY);
  columnDescriptor.setBlocksize(64 * 1024);
  columnDescriptor.setBloomFilterType(BloomType.ROW);
  columnDescriptor.setMaxVersions(10);
  tableDescriptor.addFamily(columnDescriptor);

  tableDescriptor.setMaxFileSize(regionMaxSize);
  tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY,
      ConstantSizeRegionSplitPolicy.class.getName());

  tableDescriptor.setDeferredLogFlush(true);

  regionCount = (short) Math.abs(regionCount);

  int regionRange = Short.MAX_VALUE / regionCount;
  int counter = 0;

  byte[][] splitKeys = new byte[regionCount][];
  for (byte[] splitKey : splitKeys) {
    counter = counter + regionRange;
    String key = StringUtils.leftPad(Integer.toString(counter), 5, '0');
    splitKey = Bytes.toBytes(key);
    System.out.println(" - Split: " + splitKey);
  }
  return true;
}
 
Example #15
Source File: CreateTable.java    From HBase-ToHDFS with Apache License 2.0 5 votes vote down vote up
private static void createTable(String tableName, String columnFamilyName,
    short regionCount, long regionMaxSize, HBaseAdmin admin)
    throws IOException {
  System.out.println("Creating Table: " + tableName);
  
  HTableDescriptor tableDescriptor = new HTableDescriptor(); 
  tableDescriptor.setName(Bytes.toBytes(tableName));
  
  HColumnDescriptor columnDescriptor = new HColumnDescriptor(columnFamilyName);
  
  columnDescriptor.setCompressionType(Compression.Algorithm.SNAPPY);
  columnDescriptor.setBlocksize(64 * 1024);
  columnDescriptor.setBloomFilterType(BloomType.ROW);
  
  tableDescriptor.addFamily(columnDescriptor);
  
  tableDescriptor.setMaxFileSize(regionMaxSize);
  tableDescriptor.setValue(tableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());
  
  tableDescriptor.setDeferredLogFlush(true);
  
  regionCount = (short)Math.abs(regionCount);
  
  int regionRange = Short.MAX_VALUE/regionCount;
  int counter = 0;
  
  byte[][] splitKeys = new byte[regionCount][];
  for (int i = 0 ; i < splitKeys.length; i++) {
    counter = counter + regionRange;
    String key = StringUtils.leftPad(Integer.toString(counter), 5, '0');
    splitKeys[i] = Bytes.toBytes(key); 
    System.out.println(" - Split: " + i + " '" + key + "'");
  }
  
  admin.createTable(tableDescriptor, splitKeys);
}
 
Example #16
Source File: CreateHTableJob.java    From Kylin with Apache License 2.0 4 votes vote down vote up
@Override
public int run(String[] args) throws Exception {
    Options options = new Options();

    options.addOption(OPTION_CUBE_NAME);
    options.addOption(OPTION_PARTITION_FILE_PATH);
    options.addOption(OPTION_HTABLE_NAME);
    parseOptions(options, args);

    Path partitionFilePath = new Path(getOptionValue(OPTION_PARTITION_FILE_PATH));

    String cubeName = getOptionValue(OPTION_CUBE_NAME).toUpperCase();
    KylinConfig config = KylinConfig.getInstanceFromEnv();
    CubeManager cubeMgr = CubeManager.getInstance(config);
    CubeInstance cube = cubeMgr.getCube(cubeName);
    CubeDesc cubeDesc = cube.getDescriptor();

    String tableName = getOptionValue(OPTION_HTABLE_NAME).toUpperCase();
    HTableDescriptor tableDesc = new HTableDescriptor(TableName.valueOf(tableName));
    // https://hbase.apache.org/apidocs/org/apache/hadoop/hbase/regionserver/ConstantSizeRegionSplitPolicy.html
    tableDesc.setValue(HTableDescriptor.SPLIT_POLICY, ConstantSizeRegionSplitPolicy.class.getName());
    tableDesc.setValue(IRealizationConstants.HTableTag, config.getMetadataUrlPrefix());

    Configuration conf = HBaseConfiguration.create(getConf());
    HBaseAdmin admin = new HBaseAdmin(conf);

    try {
        if (User.isHBaseSecurityEnabled(conf)) {
            // add coprocessor for bulk load
            tableDesc.addCoprocessor("org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint");
        }

        for (HBaseColumnFamilyDesc cfDesc : cubeDesc.getHBaseMapping().getColumnFamily()) {
            HColumnDescriptor cf = new HColumnDescriptor(cfDesc.getName());
            cf.setMaxVersions(1);

            if (LZOSupportnessChecker.getSupportness()) {
                logger.info("hbase will use lzo to compress data");
                cf.setCompressionType(Algorithm.LZO);
            } else {
                logger.info("hbase will not use lzo to compress data");
            }

            cf.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
            cf.setInMemory(false);
            cf.setBlocksize(4 * 1024 * 1024); // set to 4MB
            tableDesc.addFamily(cf);
        }

        byte[][] splitKeys = getSplits(conf, partitionFilePath);

        if (admin.tableExists(tableName)) {
            // admin.disableTable(tableName);
            // admin.deleteTable(tableName);
            throw new RuntimeException("HBase table " + tableName + " exists!");
        }

        DeployCoprocessorCLI.deployCoprocessor(tableDesc);

        admin.createTable(tableDesc, splitKeys);
        logger.info("create hbase table " + tableName + " done.");

        return 0;
    } catch (Exception e) {
        printUsage(options);
        e.printStackTrace(System.err);
        logger.error(e.getLocalizedMessage(), e);
        return 2;
    } finally {
        admin.close();
    }
}