Java Code Examples for org.apache.hadoop.hbase.HBaseTestingUtility

The following are top voted examples for showing how to use org.apache.hadoop.hbase.HBaseTestingUtility. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ditb   File: CompactTableAction.java   Source Code and License 6 votes vote down vote up
@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();
  boolean major = RandomUtils.nextInt(100) < majorRatio;

  LOG.info("Performing action: Compact table " + tableName + ", major=" + major);
  try {
    if (major) {
      admin.majorCompact(tableName);
    } else {
      admin.compact(tableName);
    }
  } catch (Exception ex) {
    LOG.warn("Compaction failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 
Example 2
Project: ditb   File: TestReplicationKillRS.java   Source Code and License 6 votes vote down vote up
private static Thread killARegionServer(final HBaseTestingUtility utility,
                                        final long timeout, final int rs) {
  Thread killer = new Thread() {
    public void run() {
      try {
        Thread.sleep(timeout);
        utility.getHBaseCluster().getRegionServer(rs).stop("Stopping as part of the test");
      } catch (Exception e) {
        LOG.error("Couldn't kill a region server", e);
      }
    }
  };
  killer.setDaemon(true);
  killer.start();
  return killer;
}
 
Example 3
Project: ditb   File: SplitAllRegionOfTableAction.java   Source Code and License 6 votes vote down vote up
@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();
  // Don't try the split if we're stopping
  if (context.isStopping()) {
    return;
  }


  // Don't always split. This should allow splitting of a full table later in the run
  if (ThreadLocalRandom.current().nextDouble()
      < (((double) splits) / ((double) maxFullTableSplits)) / ((double) 2)) {
    splits++;
    LOG.info("Performing action: Split all regions of  " + tableName);
    admin.split(tableName);
  } else {
    LOG.info("Skipping split of all regions.");
  }
}
 
Example 4
Project: ditb   File: OfflineMetaRebuildTestCore.java   Source Code and License 6 votes vote down vote up
@Before
public void setUpBefore() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
  TEST_UTIL.startMiniCluster(3);
  conf = TEST_UTIL.getConfiguration();
  this.connection = ConnectionFactory.createConnection(conf);
  assertEquals(0, TEST_UTIL.getHBaseAdmin().listTables().length);

  // setup the table
  table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
  tableIdx++;
  htbl = setupTable(table);
  populateTable(htbl);
  assertEquals(5, scanMeta());
  LOG.info("Table " + table + " has " + tableRowCount(conf, table)
      + " entries.");
  assertEquals(16, tableRowCount(conf, table));
  TEST_UTIL.getHBaseAdmin().disableTable(table);
  assertEquals(1, TEST_UTIL.getHBaseAdmin().listTables().length);
}
 
Example 5
Project: ditb   File: FlushTableAction.java   Source Code and License 6 votes vote down vote up
@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  // Don't try the flush if we're stopping
  if (context.isStopping()) {
    return;
  }

  LOG.info("Performing action: Flush table " + tableName);
  try {
    admin.flush(tableName);
  } catch (Exception ex) {
    LOG.warn("Flush failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 
Example 6
Project: ditb   File: TestStore.java   Source Code and License 6 votes vote down vote up
/**
 * Getting data from memstore only
 * @throws IOException
 */
@Test
public void testGet_FromMemStoreOnly() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(), qualifiers);

  //Compare
  assertCheck();
}
 
Example 7
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Grant permissions globally to the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void grantGlobal(final HBaseTestingUtility util, final String user,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.grant(null, protocol, user, actions);
        }
      }
      return null;
    }
  });
}
 
Example 8
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Revoke permissions globally from the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeGlobal(final HBaseTestingUtility util, final String user,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.revoke(null, protocol, user, actions);
        }
      }
      return null;
    }
  });
}
 
Example 9
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Revoke permissions on a table from the given user. Will wait until all active
 * AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeFromTable(final HBaseTestingUtility util, final String user,
    final TableName table, final byte[] family, final byte[] qualifier,
    final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try (Connection connection = ConnectionFactory.createConnection(util.getConfiguration())) {
        try (Table acl = connection.getTable(AccessControlLists.ACL_TABLE_NAME)) {
          BlockingRpcChannel service = acl.coprocessorService(HConstants.EMPTY_START_ROW);
          AccessControlService.BlockingInterface protocol =
              AccessControlService.newBlockingStub(service);
          ProtobufUtil.revoke(null, protocol, user, table, family, qualifier, actions);
        }
      }
      return null;
    }
  });
}
 
Example 10
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Revoke permissions on a table from the given user using AccessControlClient. Will wait until
 * all active AccessController instances have updated their permissions caches or will
 * throw an exception upon timeout (10 seconds).
 */
public static void revokeFromTableUsingAccessControlClient(final HBaseTestingUtility util,
    final Connection connection, final String user, final TableName table, final byte[] family,
    final byte[] qualifier, final Permission.Action... actions) throws Exception {
  SecureTestUtil.updateACLs(util, new Callable<Void>() {
    @Override
    public Void call() throws Exception {
      try {
        AccessControlClient.revoke(connection, table, user, family, qualifier, actions);
      } catch (Throwable t) {
        t.printStackTrace();
      }
      return null;
    }
  });
}
 
Example 11
Project: ditb   File: TestRemoveRegionMetrics.java   Source Code and License 6 votes vote down vote up
@BeforeClass
public static void startCluster() throws Exception {
  metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
  TEST_UTIL = new HBaseTestingUtility();
  conf = TEST_UTIL.getConfiguration();
  conf.getLong("hbase.splitlog.max.resubmit", 0);
  // Make the failure test faster
  conf.setInt("zookeeper.recovery.retry", 0);
  conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);

  TEST_UTIL.startMiniCluster(1, 2);
  cluster = TEST_UTIL.getHBaseCluster();

  cluster.waitForActiveAndReadyMaster();

  while (cluster.getLiveRegionServerThreads().size() < 2) {
    Threads.sleep(100);
  }
}
 
Example 12
Project: ditb   File: SecureTestUtil.java   Source Code and License 6 votes vote down vote up
public static void checkGlobalPerms(HBaseTestingUtility testUtil, Permission.Action... actions)
    throws IOException {
  Permission[] perms = new Permission[actions.length];
  for (int i = 0; i < actions.length; i++) {
    perms[i] = new Permission(actions[i]);
  }
  CheckPermissionsRequest.Builder request = CheckPermissionsRequest.newBuilder();
  for (Action a : actions) {
    request.addPermission(AccessControlProtos.Permission.newBuilder()
        .setType(AccessControlProtos.Permission.Type.Global)
        .setGlobalPermission(
            AccessControlProtos.GlobalPermission.newBuilder()
                .addAction(ProtobufUtil.toPermissionAction(a)).build()));
  }
  try(Connection conn = ConnectionFactory.createConnection(testUtil.getConfiguration());
      Table acl = conn.getTable(AccessControlLists.ACL_TABLE_NAME)) {
    BlockingRpcChannel channel = acl.coprocessorService(new byte[0]);
    AccessControlService.BlockingInterface protocol =
      AccessControlService.newBlockingStub(channel);
    try {
      protocol.checkPermissions(null, request.build());
    } catch (ServiceException se) {
      ProtobufUtil.toIOException(se);
    }
  }
}
 
Example 13
Project: ditb   File: TestTableSnapshotInputFormat.java   Source Code and License 6 votes vote down vote up
@Override
protected void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
    int numRegions, int expectedNumSplits) throws Exception {
  setupCluster();
  TableName tableName = TableName.valueOf("testWithMockedMapReduce");
  try {
    createTableAndSnapshot(
      util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions);

    JobConf job = new JobConf(util.getConfiguration());
    Path tmpTableDir = util.getRandomDir();

    TableMapReduceUtil.initTableSnapshotMapJob(snapshotName,
      COLUMNS, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
      NullWritable.class, job, false, tmpTableDir);

    // mapred doesn't support start and end keys? o.O
    verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, getStartRow(), getEndRow());

  } finally {
    util.getHBaseAdmin().deleteSnapshot(snapshotName);
    util.deleteTable(tableName);
    tearDownCluster();
  }
}
 
Example 14
Project: ditb   File: MoveRandomRegionOfTableAction.java   Source Code and License 6 votes vote down vote up
@Override
public void perform() throws Exception {
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }

  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Move random region of table " + tableName);
  List<HRegionInfo> regions = admin.getTableRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    LOG.info("Table " + tableName + " doesn't have regions to move");
    return;
  }

  HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new HRegionInfo[regions.size()]));
  LOG.debug("Unassigning region " + region.getRegionNameAsString());
  admin.unassign(region.getRegionName(), false);
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 
Example 15
Project: ditb   File: TestReplicationAdminWithTwoDifferentZKClusters.java   Source Code and License 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.setBoolean(HConstants.REPLICATION_ENABLE_KEY, HConstants.REPLICATION_ENABLE_DEFAULT);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniCluster();
  admin = new ReplicationAdmin(conf1);

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  conf2.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, 2182);

  utility2 = new HBaseTestingUtility(conf2);
  utility2.startMiniCluster();

  ReplicationPeerConfig config = new ReplicationPeerConfig();
  config.setClusterKey(utility2.getClusterKey());
  admin.addPeer(peerId, config, null);

  HTableDescriptor table = new HTableDescriptor(tableName);
  HColumnDescriptor fam = new HColumnDescriptor(famName);
  fam.setScope(HConstants.REPLICATION_SCOPE_GLOBAL);
  table.addFamily(fam);

  utility1.getHBaseAdmin().createTable(table, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  utility1.waitUntilAllRegionsAssigned(tableName);
}
 
Example 16
Project: ditb   File: TestClientOperationInterrupt.java   Source Code and License 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = HBaseConfiguration.create();
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessor.class.getName());
  util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  Admin admin = util.getHBaseAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }
  util.createTable(tableName, new byte[][]{dummy, test});

  Table ht = new HTable(conf, tableName);
  Put p = new Put(row1);
  p.add(dummy, dummy, dummy);
  ht.put(p);
}
 
Example 17
Project: ditb   File: TestRegionLocationFinder.java   Source Code and License 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  cluster = TEST_UTIL.startMiniCluster(1, ServerNum);
  table = TEST_UTIL.createTable(tableName, FAMILY, HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE);
  TEST_UTIL.waitTableAvailable(tableName, 1000);
  TEST_UTIL.loadTable(table, FAMILY);

  for (int i = 0; i < ServerNum; i++) {
    HRegionServer server = cluster.getRegionServer(i);
    for (Region region : server.getOnlineRegions(tableName)) {
      region.flush(true);
    }
  }

  finder.setConf(TEST_UTIL.getConfiguration());
  finder.setServices(cluster.getMaster());
  finder.setClusterStatus(cluster.getMaster().getClusterStatus());
}
 
Example 18
Project: ditb   File: TestEnableTableHandler.java   Source Code and License 6 votes vote down vote up
public static void createTable(HBaseTestingUtility testUtil, HBaseAdmin admin,
  HTableDescriptor htd, byte [][] splitKeys)
throws Exception {
  // NOTE: We need a latch because admin is not sync,
  // so the postOp coprocessor method may be called after the admin operation returned.
  MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
    .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
  observer.tableCreationLatch = new CountDownLatch(1);
  if (splitKeys != null) {
    admin.createTable(htd, splitKeys);
  } else {
    admin.createTable(htd);
  }
  observer.tableCreationLatch.await();
  observer.tableCreationLatch = null;
  testUtil.waitUntilAllRegionsAssigned(htd.getTableName());
}
 
Example 19
Project: ditb   File: TestEnableTableHandler.java   Source Code and License 6 votes vote down vote up
public static void deleteTable(HBaseTestingUtility testUtil, HBaseAdmin admin,
  TableName tableName)
throws Exception {
  // NOTE: We need a latch because admin is not sync,
  // so the postOp coprocessor method may be called after the admin operation returned.
  MasterSyncObserver observer = (MasterSyncObserver)testUtil.getHBaseCluster().getMaster()
    .getMasterCoprocessorHost().findCoprocessor(MasterSyncObserver.class.getName());
  observer.tableDeletionLatch = new CountDownLatch(1);
  try {
    admin.disableTable(tableName);
  } catch (Exception e) {
    LOG.debug("Table: " + tableName + " already disabled, so just deleting it.");
  }
  admin.deleteTable(tableName);
  observer.tableDeletionLatch.await();
  observer.tableDeletionLatch = null;
}
 
Example 20
Project: ditb   File: ProcessBasedLocalHBaseCluster.java   Source Code and License 6 votes vote down vote up
/**
 * Constructor. Modifies the passed configuration.
 * @param hbaseHome the top directory of the HBase source tree
 */
public ProcessBasedLocalHBaseCluster(Configuration conf,
    int numDataNodes, int numRegionServers) {
  this.conf = conf;
  this.hbaseHome = HBaseHomePath.getHomePath();
  this.numMasters = 1;
  this.numRegionServers = numRegionServers;
  this.workDir = hbaseHome + "/target/local_cluster";
  this.numDataNodes = numDataNodes;

  hbaseDaemonScript = hbaseHome + "/bin/hbase-daemon.sh";
  zkClientPort = HBaseTestingUtility.randomFreePort();

  this.rsPorts = sortedPorts(numRegionServers);
  this.masterPorts = sortedPorts(numMasters);

  conf.set(HConstants.ZOOKEEPER_QUORUM, HConstants.LOCALHOST);
  conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClientPort);
}
 
Example 21
Project: ditb   File: TestFSUtils.java   Source Code and License 6 votes vote down vote up
@Test
public void testVersion() throws DeserializationException, IOException {
  HBaseTestingUtility htu = new HBaseTestingUtility();
  final FileSystem fs = htu.getTestFileSystem();
  final Path rootdir = htu.getDataTestDir();
  assertNull(FSUtils.getVersion(fs, rootdir));
  // Write out old format version file.  See if we can read it in and convert.
  Path versionFile = new Path(rootdir, HConstants.VERSION_FILE_NAME);
  FSDataOutputStream s = fs.create(versionFile);
  final String version = HConstants.FILE_SYSTEM_VERSION;
  s.writeUTF(version);
  s.close();
  assertTrue(fs.exists(versionFile));
  FileStatus [] status = fs.listStatus(versionFile);
  assertNotNull(status);
  assertTrue(status.length > 0);
  String newVersion = FSUtils.getVersion(fs, rootdir);
  assertEquals(version.length(), newVersion.length());
  assertEquals(version, newVersion);
  // File will have been converted. Exercise the pb format
  assertEquals(version, FSUtils.getVersion(fs, rootdir));
  FSUtils.checkVersion(fs, rootdir, true);
}
 
Example 22
Project: ditb   File: TestTableSnapshotInputFormat.java   Source Code and License 6 votes vote down vote up
@Override
public void testWithMockedMapReduce(HBaseTestingUtility util, String snapshotName,
    int numRegions, int expectedNumSplits) throws Exception {
  setupCluster();
  TableName tableName = TableName.valueOf("testWithMockedMapReduce");
  try {
    createTableAndSnapshot(
      util, tableName, snapshotName, getStartRow(), getEndRow(), numRegions);

    Job job = new Job(util.getConfiguration());
    Path tmpTableDir = util.getRandomDir();
    Scan scan = new Scan(getStartRow(), getEndRow()); // limit the scan

    TableMapReduceUtil.initTableSnapshotMapperJob(snapshotName,
        scan, TestTableSnapshotMapper.class, ImmutableBytesWritable.class,
        NullWritable.class, job, false, tmpTableDir);

    verifyWithMockedMapReduce(job, numRegions, expectedNumSplits, getStartRow(), getEndRow());

  } finally {
    util.getHBaseAdmin().deleteSnapshot(snapshotName);
    util.deleteTable(tableName);
    tearDownCluster();
  }
}
 
Example 23
Project: ditb   File: TestFSUtils.java   Source Code and License 5 votes vote down vote up
@Test
public void testDeleteAndExists() throws Exception {
  HBaseTestingUtility htu = new HBaseTestingUtility();
  Configuration conf = htu.getConfiguration();
  conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true);
  FileSystem fs = FileSystem.get(conf);
  FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY);
  // then that the correct file is created
  String file = UUID.randomUUID().toString();
  Path p = new Path(htu.getDataTestDir(), "temptarget" + File.separator + file);
  Path p1 = new Path(htu.getDataTestDir(), "temppath" + File.separator + file);
  try {
    FSDataOutputStream out = FSUtils.create(conf, fs, p, perms, null);
    out.close();
    assertTrue("The created file should be present", FSUtils.isExists(fs, p));
    // delete the file with recursion as false. Only the file will be deleted.
    FSUtils.delete(fs, p, false);
    // Create another file
    FSDataOutputStream out1 = FSUtils.create(conf, fs, p1, perms, null);
    out1.close();
    // delete the file with recursion as false. Still the file only will be deleted
    FSUtils.delete(fs, p1, true);
    assertFalse("The created file should be present", FSUtils.isExists(fs, p1));
    // and then cleanup
  } finally {
    FSUtils.delete(fs, p, true);
    FSUtils.delete(fs, p1, true);
  }
}
 
Example 24
Project: ignite-hbase   File: HBaseCacheStoreTest.java   Source Code and License 5 votes vote down vote up
@BeforeClass
public static void setupClass() throws Exception {
  utility = new HBaseTestingUtility();
  Path dataTestDir = utility.getDataTestDir().getParent();
  int length = dataTestDir.toString().length();
  if (length > TEST_DIRECTORY_MAX_LENGTH) {
    System.err.println(TEST_DIRECTORY_INVALID_MESSAGE);
    System.err.println("Current HBase test directory: " + dataTestDir);
    throw new RuntimeException(TEST_DIRECTORY_INVALID_MESSAGE);
  }
  utility.startMiniCluster();
}
 
Example 25
Project: dremio-oss   File: HBaseTestsSuite.java   Source Code and License 5 votes vote down vote up
@BeforeClass
public static void initCluster() throws Exception {
  assumeNonMaprProfile();
  if (initCount.get() == 0) {
    synchronized (HBaseTestsSuite.class) {
      if (initCount.get() == 0) {
        conf = HBaseConfiguration.create();
        conf.set(HConstants.HBASE_CLIENT_INSTANCE_ID, "dremio-hbase-unit-tests-client");
        if (IS_DEBUG) {
          conf.set("hbase.client.scanner.timeout.period","10000000");
        }

        if (manageHBaseCluster) {
          logger.info("Starting HBase mini cluster.");
          UTIL = new HBaseTestingUtility(conf);
          UTIL.startMiniZKCluster();
          String old_home = System.getProperty("user.home");
          System.setProperty("user.home", UTIL.getDataTestDir().toString());
          UTIL.startMiniHBaseCluster(1, 1);
          System.setProperty("user.home", old_home);
          hbaseClusterCreated = true;
          logger.info("HBase mini cluster started. Zookeeper port: '{}'", getZookeeperPort());
        }

        conn = ConnectionFactory.createConnection(conf);
        admin = conn.getAdmin();

        if (createTables || !tablesExist()) {
          createTestTables();
          tablesCreated = true;
        }
        initCount.incrementAndGet();
        return;
      }
    }
  }
  initCount.incrementAndGet();
}
 
Example 26
Project: ditb   File: TestStore.java   Source Code and License 5 votes vote down vote up
/**
 * Getting data from memstore and files
 * @throws IOException
 */
@Test
public void testGet_FromMemStoreAndFiles() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(), qualifiers);

  //Need to sort the result since multiple files
  Collections.sort(result, KeyValue.COMPARATOR);

  //Compare
  assertCheck();
}
 
Example 27
Project: ditb   File: HFileArchiveTestingUtil.java   Source Code and License 5 votes vote down vote up
public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
    byte[] storeName) throws IOException {
  byte[] table = Bytes.toBytes(tableName);
  // get the RS and region serving our table
  List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
  HRegion region = servingRegions.get(0);

  // check that we actually have some store files that were archived
  Store store = region.getStore(storeName);
  return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
}
 
Example 28
Project: ditb   File: TestStore.java   Source Code and License 5 votes vote down vote up
/**
 * Getting data from files only
 * @throws IOException
 */
@Test
public void testGet_FromFilesOnly() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null));
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null));
  //flush
  flush(3);

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(),
      qualifiers);
  //this.store.get(get, qualifiers, result);

  //Need to sort the result since multiple files
  Collections.sort(result, KeyValue.COMPARATOR);

  //Compare
  assertCheck();
}
 
Example 29
Project: ditb   File: TestRegionReplicaReplicationEndpoint.java   Source Code and License 5 votes vote down vote up
@Test (timeout = 240000)
public void testRegionReplicaWithoutMemstoreReplication() throws Exception {
  int regionReplication = 3;
  TableName tableName = TableName.valueOf("testRegionReplicaWithoutMemstoreReplication");
  HTableDescriptor htd = HTU.createTableDescriptor(tableName.toString());
  htd.setRegionReplication(regionReplication);
  htd.setRegionMemstoreReplication(false);
  HTU.getHBaseAdmin().createTable(htd);

  Connection connection = ConnectionFactory.createConnection(HTU.getConfiguration());
  Table table = connection.getTable(tableName);
  try {
    // write data to the primary. The replicas should not receive the data
    final int STEP = 100;
    for (int i = 0; i < 3; ++i) {
      final int startRow = i * STEP;
      final int endRow = (i + 1) * STEP;
      LOG.info("Writing data from " + startRow + " to " + endRow);
      HTU.loadNumericRows(table, HBaseTestingUtility.fam1, startRow, endRow);
      verifyReplication(tableName, regionReplication, startRow, endRow, false);

      // Flush the table, now the data should show up in the replicas
      LOG.info("flushing table");
      HTU.flush(tableName);
      verifyReplication(tableName, regionReplication, 0, endRow, true);
    }
  } finally {
    table.close();
    connection.close();
  }
}
 
Example 30
Project: ditb   File: TestWALReplay.java   Source Code and License 5 votes vote down vote up
private WAL createWAL(final Configuration c) throws IOException {
  FSHLog wal = new FSHLog(FileSystem.get(c), hbaseRootDir, logName, c);
  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
  return wal;
}
 
Example 31
Project: ditb   File: SplitRandomRegionOfTableAction.java   Source Code and License 5 votes vote down vote up
@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Split random region of table " + tableName);
  List<HRegionInfo> regions = admin.getTableRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    LOG.info("Table " + tableName + " doesn't have regions to split");
    return;
  }
  // Don't try the split if we're stopping
  if (context.isStopping()) {
    return;
  }

  HRegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
      regions.toArray(new HRegionInfo[regions.size()]));
  LOG.debug("Splitting region " + region.getRegionNameAsString());
  try {
    admin.splitRegion(region.getRegionName());
  } catch (Exception ex) {
    LOG.warn("Split failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 
Example 32
Project: ditb   File: SnapshotTestingUtils.java   Source Code and License 5 votes vote down vote up
public static void deleteArchiveDirectory(final HBaseTestingUtility util)
    throws IOException {
  // Ensure the archiver to be empty
  MasterFileSystem mfs = util.getMiniHBaseCluster().getMaster().getMasterFileSystem();
  Path archiveDir = new Path(mfs.getRootDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);
  mfs.getFileSystem().delete(archiveDir, true);
}
 
Example 33
Project: ditb   File: ProcessBasedLocalHBaseCluster.java   Source Code and License 5 votes vote down vote up
public void startHBase() throws IOException {
  startDaemonLogTailer();
  cleanupOldState();

  // start ZK
  LOG.info("Starting ZooKeeper on port " + zkClientPort);
  startZK();

  HBaseTestingUtility.waitForHostPort(HConstants.LOCALHOST, zkClientPort);

  for (int masterPort : masterPorts) {
    startMaster(masterPort);
  }

  ZKUtil.waitForBaseZNode(conf);

  for (int rsPort : rsPorts) {
    startRegionServer(rsPort);
  }

  LOG.info("Waiting for HBase startup by scanning META");
  int attemptsLeft = 10;
  while (attemptsLeft-- > 0) {
    try {
      new HTable(conf, TableName.META_TABLE_NAME);
    } catch (Exception e) {
      LOG.info("Waiting for HBase to startup. Retries left: " + attemptsLeft,
          e);
      Threads.sleep(1000);
    }
  }

  LOG.info("Process-based HBase Cluster with " + numRegionServers +
      " region servers up and running... \n\n");
}
 
Example 34
Project: ditb   File: TestFileLink.java   Source Code and License 5 votes vote down vote up
/**
 * Test, on a local filesystem, that the FileLink is still readable
 * even when the current file gets renamed.
 */
@Test
public void testLocalLinkReadDuringRename() throws IOException {
  HBaseTestingUtility testUtil = new HBaseTestingUtility();
  FileSystem fs = testUtil.getTestFileSystem();
  assertEquals("file", fs.getUri().getScheme());
  testLinkReadDuringRename(fs, testUtil.getDataTestDir());
}
 
Example 35
Project: ditb   File: MockServer.java   Source Code and License 5 votes vote down vote up
/**
 * @param htu Testing utility to use
 * @param zkw If true, create a zkw.
 * @throws ZooKeeperConnectionException
 * @throws IOException
 */
public MockServer(final HBaseTestingUtility htu, final boolean zkw)
throws ZooKeeperConnectionException, IOException {
  this.htu = htu;
  this.zk = zkw?
    new ZooKeeperWatcher(htu.getConfiguration(), NAME.toString(), this, true):
    null;
}
 
Example 36
Project: ditb   File: ChangeBloomFilterAction.java   Source Code and License 5 votes vote down vote up
@Override
public void perform() throws Exception {
  Random random = new Random();
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Change bloom filter on all columns of table "
      + tableName);
  HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName);
  HColumnDescriptor[] columnDescriptors = tableDescriptor.getColumnFamilies();

  if (columnDescriptors == null || columnDescriptors.length == 0) {
    return;
  }

  final BloomType[] bloomArray = BloomType.values();
  final int bloomArraySize = bloomArray.length;

  for (HColumnDescriptor descriptor : columnDescriptors) {
    int bloomFilterIndex = random.nextInt(bloomArraySize);
    LOG.debug("Performing action: About to set bloom filter type to "
        + bloomArray[bloomFilterIndex] + " on column "
        + descriptor.getNameAsString() + " of table " + tableName);
    descriptor.setBloomFilterType(bloomArray[bloomFilterIndex]);
    LOG.debug("Performing action: Just set bloom filter type to "
        + bloomArray[bloomFilterIndex] + " on column "
        + descriptor.getNameAsString() + " of table " + tableName);
  }

  // Don't try the modify if we're stopping
  if (context.isStopping()) {
    return;
  }
  admin.modifyTable(tableName, tableDescriptor);
}
 
Example 37
Project: ditb   File: SecureTestUtil.java   Source Code and License 5 votes vote down vote up
public static void checkTablePerms(HBaseTestingUtility testUtil, TableName table, byte[] family,
    byte[] column, Permission.Action... actions) throws IOException {
  Permission[] perms = new Permission[actions.length];
  for (int i = 0; i < actions.length; i++) {
    perms[i] = new TablePermission(table, family, column, actions[i]);
  }
  checkTablePerms(testUtil, table, perms);
}
 
Example 38
Project: ditb   File: SnapshotTestingUtils.java   Source Code and License 5 votes vote down vote up
public static void confirmSnapshotValid(HBaseTestingUtility testUtil,
    SnapshotDescription snapshotDescriptor, TableName tableName, byte[] family)
    throws IOException {
  MasterFileSystem mfs = testUtil.getHBaseCluster().getMaster().getMasterFileSystem();
  confirmSnapshotValid(snapshotDescriptor, tableName, family,
      mfs.getRootDir(), testUtil.getHBaseAdmin(), mfs.getFileSystem());
}
 
Example 39
Project: ditb   File: TestStore.java   Source Code and License 5 votes vote down vote up
/**
 * Test for hbase-1686.
 * @throws IOException
 */
@Test
public void testEmptyStoreFile() throws IOException {
  init(this.name.getMethodName());
  // Write a store file.
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null));
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null));
  flush(1);
  // Now put in place an empty store file.  Its a little tricky.  Have to
  // do manually with hacked in sequence id.
  StoreFile f = this.store.getStorefiles().iterator().next();
  Path storedir = f.getPath().getParent();
  long seqid = f.getMaxSequenceId();
  Configuration c = HBaseConfiguration.create();
  FileSystem fs = FileSystem.get(c);
  HFileContext meta = new HFileContextBuilder().withBlockSize(BLOCKSIZE_SMALL).build();
  StoreFile.Writer w = new StoreFile.WriterBuilder(c, new CacheConfig(c),
      fs)
          .withOutputDir(storedir)
          .withFileContext(meta)
          .build();
  w.appendMetadata(seqid + 1, false);
  w.close();
  this.store.close();
  // Reopen it... should pick up two files
  this.store = new HStore(this.store.getHRegion(), this.store.getFamily(), c);
  Assert.assertEquals(2, this.store.getStorefilesCount());

  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(),
      qualifiers);
  Assert.assertEquals(1, result.size());
}
 
Example 40
Project: ditb   File: TestWALLockup.java   Source Code and License 5 votes vote down vote up
@Before
public void setup() throws IOException {
  TEST_UTIL = HBaseTestingUtility.createLocalHTU();
  CONF = TEST_UTIL.getConfiguration();
  // Disable block cache.
  CONF.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0f);
  dir = TEST_UTIL.getDataTestDir("TestHRegion").toString();
  tableName = TableName.valueOf(name.getMethodName());
}
 
Example 41
Project: ditb   File: TestRegionServerNoMaster.java   Source Code and License 5 votes vote down vote up
public static void closeRegion(HBaseTestingUtility HTU, HRegionServer rs, HRegionInfo hri)
    throws Exception {
  ZKAssign.createNodeClosing(HTU.getZooKeeperWatcher(), hri, rs.getServerName());
  AdminProtos.CloseRegionRequest crr = RequestConverter.buildCloseRegionRequest(
    rs.getServerName(), hri.getEncodedName(), true);
  AdminProtos.CloseRegionResponse responseClose = rs.rpcServices.closeRegion(null, crr);
  Assert.assertTrue(responseClose.getClosed());
  checkRegionIsClosed(HTU, rs, hri);
  ZKAssign.deleteClosedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(), null);
}
 
Example 42
Project: ditb   File: TestTableSnapshotScanner.java   Source Code and License 5 votes vote down vote up
private void testScanner(HBaseTestingUtility util, String snapshotName, int numRegions,
    boolean shutdownCluster) throws Exception {
  setupCluster();
  TableName tableName = TableName.valueOf("testScanner");
  try {
    createTableAndSnapshot(util, tableName, snapshotName, numRegions);

    if (shutdownCluster) {
      util.shutdownMiniHBaseCluster();
    }

    Path restoreDir = util.getDataTestDirOnTestFS(snapshotName);
    Scan scan = new Scan(bbb, yyy); // limit the scan

    TableSnapshotScanner scanner = new TableSnapshotScanner(UTIL.getConfiguration(), restoreDir,
      snapshotName, scan);

    verifyScanner(scanner, bbb, yyy);
    scanner.close();
  } finally {
    if (!shutdownCluster) {
      util.getHBaseAdmin().deleteSnapshot(snapshotName);
      util.deleteTable(tableName);
      tearDownCluster();
    }
  }
}
 
Example 43
Project: ditb   File: TruncateTableAction.java   Source Code and License 5 votes vote down vote up
@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  // Don't try the truncate if we're stopping
  if (context.isStopping()) {
    return;
  }

  boolean preserveSplits = random.nextBoolean();
  LOG.info("Performing action: Truncate table " + tableName.getNameAsString() +
           "preserve splits " + preserveSplits);
  admin.truncateTable(tableName, preserveSplits);
}
 
Example 44
Project: ditb   File: TestMultiRespectsLimits.java   Source Code and License 5 votes vote down vote up
@Test
public void testMultiLimits() throws Exception {
  final TableName name = TableName.valueOf("testMultiLimits");
  Table t = TEST_UTIL.createTable(name, FAMILY);
  TEST_UTIL.loadTable(t, FAMILY, false);

  // Split the table to make sure that the chunking happens accross regions.
  try (final Admin admin = TEST_UTIL.getHBaseAdmin()) {
    admin.split(name);
    TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
      @Override
      public boolean evaluate() throws Exception {
        return admin.getTableRegions(name).size() > 1;
      }
    });
  }
  List<Get> gets = new ArrayList<>(MAX_SIZE);

  for (int i = 0; i < MAX_SIZE; i++) {
    gets.add(new Get(HBaseTestingUtility.ROWS[i]));
  }

  RpcServerInterface rpcServer = TEST_UTIL.getHBaseCluster().getRegionServer(0).getRpcServer();
  BaseSource s = rpcServer.getMetrics().getMetricsSource();
  long startingExceptions = METRICS_ASSERT.getCounter("exceptions", s);
  long startingMultiExceptions = METRICS_ASSERT.getCounter("exceptions.multiResponseTooLarge", s);

  Result[] results = t.get(gets);
  assertEquals(MAX_SIZE, results.length);

  // Cells from TEST_UTIL.loadTable have a length of 27.
  // Multiplying by less than that gives an easy lower bound on size.
  // However in reality each kv is being reported as much higher than that.
  METRICS_ASSERT.assertCounterGt("exceptions",
      startingExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
  METRICS_ASSERT.assertCounterGt("exceptions.multiResponseTooLarge",
      startingMultiExceptions + ((MAX_SIZE * 25) / MAX_SIZE), s);
}
 
Example 45
Project: ditb   File: TestFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testGetStartEndKeysWithRegionReplicas() throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testGetStartEndKeys"));
  HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
  htd.addFamily(fam);
  byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
  HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
  admin.createTable(htd, KEYS);
  List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());

  HRegionLocator locator =
      (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
  for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
    List<RegionLocations> regionLocations = new ArrayList<RegionLocations>();

    // mock region locations coming from meta with multiple replicas
    for (HRegionInfo region : regions) {
      HRegionLocation[] arr = new HRegionLocation[regionReplication];
      for (int i = 0; i < arr.length; i++) {
        arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
      }
      regionLocations.add(new RegionLocations(arr));
    }

    Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);

    assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);

    for (int i = 0; i < KEYS.length + 1; i++) {
      byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
      byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
      assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
      assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
    }
  }
}
 
Example 46
Project: ditb   File: TestWALReplay.java   Source Code and License 5 votes vote down vote up
private MockWAL createMockWAL() throws IOException {
  MockWAL wal = new MockWAL(fs, hbaseRootDir, logName, conf);
  // Set down maximum recovery so we dfsclient doesn't linger retrying something
  // long gone.
  HBaseTestingUtility.setMaxRecoveryErrorCount(wal.getOutputStream(), 1);
  return wal;
}
 
Example 47
Project: ditb   File: TestCatalogJanitor.java   Source Code and License 5 votes vote down vote up
private String setRootDirAndCleanIt(final HBaseTestingUtility htu,
    final String subdir)
throws IOException {
  Path testdir = htu.getDataTestDir(subdir);
  FileSystem fs = FileSystem.get(htu.getConfiguration());
  if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true));
  FSUtils.setRootDir(htu.getConfiguration(), testdir);
  return FSUtils.getRootDir(htu.getConfiguration()).toString();
}
 
Example 48
Project: ditb   File: TestRegionServerNoMaster.java   Source Code and License 5 votes vote down vote up
public static void checkRegionIsOpened(HBaseTestingUtility HTU, HRegionServer rs,
    HRegionInfo hri) throws Exception {
  while (!rs.getRegionsInTransitionInRS().isEmpty()) {
    Thread.sleep(1);
  }

  Assert.assertTrue(rs.getRegion(hri.getRegionName()).isAvailable());

  Assert.assertTrue(
    ZKAssign.deleteOpenedNode(HTU.getZooKeeperWatcher(), hri.getEncodedName(),
      rs.getServerName()));
}
 
Example 49
Project: ditb   File: TestSplitLogManager.java   Source Code and License 5 votes vote down vote up
@Before
public void setup() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  TEST_UTIL.startMiniZKCluster();
  conf = TEST_UTIL.getConfiguration();
  // Use a different ZK wrapper instance for each tests.
  zkw =
      new ZooKeeperWatcher(conf, "split-log-manager-tests" + UUID.randomUUID().toString(), null);
  ds = new DummyServer(zkw, conf);

  ZKUtil.deleteChildrenRecursively(zkw, zkw.baseZNode);
  ZKUtil.createAndFailSilent(zkw, zkw.baseZNode);
  assertTrue(ZKUtil.checkExists(zkw, zkw.baseZNode) != -1);
  LOG.debug(zkw.baseZNode + " created");
  ZKUtil.createAndFailSilent(zkw, zkw.splitLogZNode);
  assertTrue(ZKUtil.checkExists(zkw, zkw.splitLogZNode) != -1);
  LOG.debug(zkw.splitLogZNode + " created");

  stopped = false;
  resetCounters();

  // By default, we let the test manage the error as before, so the server
  // does not appear as dead from the master point of view, only from the split log pov.
  Mockito.when(sm.isServerOnline(Mockito.any(ServerName.class))).thenReturn(true);
  Mockito.when(master.getServerManager()).thenReturn(sm);

  to = 12000;
  conf.setInt(HConstants.HBASE_SPLITLOG_MANAGER_TIMEOUT, to);
  conf.setInt("hbase.splitlog.manager.unassigned.timeout", 2 * to);

  conf.setInt("hbase.splitlog.manager.timeoutmonitor.period", 100);
  to = to + 4 * 100;

  this.mode =
      (conf.getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false) ? RecoveryMode.LOG_REPLAY
          : RecoveryMode.LOG_SPLITTING);
}
 
Example 50
Project: ditb   File: TestSeekBeforeWithReverseScan.java   Source Code and License 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  TableName tableName = TableName.valueOf(getClass().getSimpleName());
  HTableDescriptor htd = new HTableDescriptor(tableName);
  htd.addFamily(new HColumnDescriptor(cfName).setDataBlockEncoding(DataBlockEncoding.FAST_DIFF));
  HRegionInfo info = new HRegionInfo(tableName, null, null, false);
  Path path = testUtil.getDataTestDir(getClass().getSimpleName());
  region = HBaseTestingUtility.createRegionAndWAL(info, path, testUtil.getConfiguration(), htd);
}
 
Example 51
Project: ditb   File: TestMasterFailoverBalancerPersistence.java   Source Code and License 5 votes vote down vote up
/**
 * Test that if the master fails, the load balancer maintains its
 * state (running or not) when the next master takes over
 *
 * @throws Exception
 */
@Test(timeout = 240000)
public void testMasterFailoverBalancerPersistence() throws Exception {
  final int NUM_MASTERS = 3;
  final int NUM_RS = 1;

  // Start the cluster
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

  assertTrue(cluster.waitForActiveAndReadyMaster());
  HMaster active = cluster.getMaster();
  // check that the balancer is on by default for the active master
  ClusterStatus clusterStatus = active.getClusterStatus();
  assertTrue(clusterStatus.isBalancerOn());

  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is still running on new master
  clusterStatus = active.getClusterStatus();
  assertTrue(clusterStatus.isBalancerOn());

  // turn off the load balancer
  active.balanceSwitch(false);

  // once more, kill active master and wait for new active master to show up
  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is not running on the new master
  clusterStatus = active.getClusterStatus();
  assertFalse(clusterStatus.isBalancerOn());

  // Stop the cluster
  TEST_UTIL.shutdownMiniCluster();
}
 
Example 52
Project: ditb   File: TestMasterFileSystem.java   Source Code and License 5 votes vote down vote up
@Test
public void testRemoveStaleRecoveringRegionsDuringMasterInitialization() throws Exception {
  // this test is for when distributed log replay is enabled
  if (!UTIL.getConfiguration().getBoolean(HConstants.DISTRIBUTED_LOG_REPLAY_KEY, false)) return;
  
  LOG.info("Starting testRemoveStaleRecoveringRegionsDuringMasterInitialization");
  HMaster master = UTIL.getMiniHBaseCluster().getMaster();
  MasterFileSystem fs = master.getMasterFileSystem();

  String failedRegion = "failedRegoin1";
  String staleRegion = "staleRegion";
  ServerName inRecoveryServerName = ServerName.valueOf("mgr,1,1");
  ServerName previouselyFaildServerName = ServerName.valueOf("previous,1,1");
  String walPath = "/hbase/data/.logs/" + inRecoveryServerName.getServerName()
      + "-splitting/test";
  // Create a ZKW to use in the test
  ZooKeeperWatcher zkw = HBaseTestingUtility.getZooKeeperWatcher(UTIL);
  zkw.getRecoverableZooKeeper().create(ZKSplitLog.getEncodedNodeName(zkw, walPath),
    new SplitLogTask.Owned(inRecoveryServerName, fs.getLogRecoveryMode()).toByteArray(), 
      Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
  String staleRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, staleRegion);
  ZKUtil.createWithParents(zkw, staleRegionPath);
  String inRecoveringRegionPath = ZKUtil.joinZNode(zkw.recoveringRegionsZNode, failedRegion);
  inRecoveringRegionPath = ZKUtil.joinZNode(inRecoveringRegionPath, 
    inRecoveryServerName.getServerName());
  ZKUtil.createWithParents(zkw, inRecoveringRegionPath);
  Set<ServerName> servers = new HashSet<ServerName>();
  servers.add(previouselyFaildServerName);
  fs.removeStaleRecoveringRegionsFromZK(servers);

  // verification
  assertFalse(ZKUtil.checkExists(zkw, staleRegionPath) != -1);
  assertTrue(ZKUtil.checkExists(zkw, inRecoveringRegionPath) != -1);
    
  ZKUtil.deleteChildrenRecursively(zkw, zkw.recoveringRegionsZNode);
  ZKUtil.deleteChildrenRecursively(zkw, zkw.splitLogZNode);
  zkw.close();
}
 
Example 53
Project: ditb   File: TestRegionServerReportForDuty.java   Source Code and License 5 votes vote down vote up
@Before
public void setUp() throws Exception {
  testUtil = new HBaseTestingUtility();
  testUtil.startMiniDFSCluster(1);
  testUtil.startMiniZKCluster(1);
  testUtil.createRootDir();
  cluster = new LocalHBaseCluster(testUtil.getConfiguration(), 0, 0);
}
 
Example 54
Project: ditb   File: ChangeSplitPolicyAction.java   Source Code and License 5 votes vote down vote up
@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getHBaseAdmin();

  LOG.info("Performing action: Change split policy of table " + tableName);
  HTableDescriptor tableDescriptor = admin.getTableDescriptor(tableName);
  String chosenPolicy = possiblePolicies[random.nextInt(possiblePolicies.length)];
  tableDescriptor.setRegionSplitPolicyClassName(chosenPolicy);
  LOG.info("Changing "  + tableName + " split policy to " + chosenPolicy);
  admin.modifyTable(tableName, tableDescriptor);
}
 
Example 55
Project: ditb   File: TestRegionServerNoMaster.java   Source Code and License 5 votes vote down vote up
public static void openRegion(HBaseTestingUtility HTU, HRegionServer rs, HRegionInfo hri)
    throws Exception {
  ZKAssign.createNodeOffline(HTU.getZooKeeperWatcher(), hri, rs.getServerName());
  // first version is '0'
  AdminProtos.OpenRegionRequest orr =
      RequestConverter.buildOpenRegionRequest(rs.getServerName(), hri, 0, null, null);
  AdminProtos.OpenRegionResponse responseOpen = rs.rpcServices.openRegion(null, orr);

  Assert.assertTrue(responseOpen.getOpeningStateCount() == 1);
  Assert.assertTrue(responseOpen.getOpeningState(0).
      equals(AdminProtos.OpenRegionResponse.RegionOpeningState.OPENED));


  checkRegionIsOpened(HTU, rs, hri);
}
 
Example 56
Project: ditb   File: TestImportTSVWithTTLs.java   Source Code and License 5 votes vote down vote up
protected static Tool doMROnTableTest(HBaseTestingUtility util, String family, String data,
    String[] args, int valueMultiplier) throws Exception {
  TableName table = TableName.valueOf(args[args.length - 1]);
  Configuration conf = new Configuration(util.getConfiguration());

  // populate input file
  FileSystem fs = FileSystem.get(conf);
  Path inputPath = fs.makeQualified(new Path(util
      .getDataTestDirOnTestFS(table.getNameAsString()), "input.dat"));
  FSDataOutputStream op = fs.create(inputPath, true);
  op.write(Bytes.toBytes(data));
  op.close();
  LOG.debug(String.format("Wrote test data to file: %s", inputPath));

  if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {
    LOG.debug("Forcing combiner.");
    conf.setInt("mapreduce.map.combine.minspills", 1);
  }

  // run the import
  List<String> argv = new ArrayList<String>(Arrays.asList(args));
  argv.add(inputPath.toString());
  Tool tool = new ImportTsv();
  LOG.debug("Running ImportTsv with arguments: " + argv);
  try {
    // Job will fail if observer rejects entries without TTL
    assertEquals(0, ToolRunner.run(conf, tool, argv.toArray(args)));
  } finally {
    // Clean up
    if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {
      LOG.debug("Deleting test subdirectory");
      util.cleanupDataTestDirOnTestFS(table.getNameAsString());
    }
  }

  return tool;
}
 
Example 57
Project: ditb   File: TestTableSnapshotInputFormat.java   Source Code and License 5 votes vote down vote up
@Override
protected void testWithMapReduceImpl(HBaseTestingUtility util, TableName tableName,
    String snapshotName, Path tableDir, int numRegions, int expectedNumSplits,
    boolean shutdownCluster) throws Exception {
  doTestWithMapReduce(util, tableName, snapshotName, getStartRow(), getEndRow(), tableDir,
    numRegions, expectedNumSplits, shutdownCluster);
}
 
Example 58
Project: ditb   File: TestFSUtils.java   Source Code and License 5 votes vote down vote up
@Test
public void testRenameAndSetModifyTime() throws Exception {
  HBaseTestingUtility htu = new HBaseTestingUtility();
  Configuration conf = htu.getConfiguration();

  MiniDFSCluster cluster = htu.startMiniDFSCluster(1);
  assertTrue(FSUtils.isHDFS(conf));

  FileSystem fs = FileSystem.get(conf);
  Path testDir = htu.getDataTestDirOnTestFS("testArchiveFile");

  String file = UUID.randomUUID().toString();
  Path p = new Path(testDir, file);

  FSDataOutputStream out = fs.create(p);
  out.close();
  assertTrue("The created file should be present", FSUtils.isExists(fs, p));

  long expect = System.currentTimeMillis() + 1000;
  assertNotEquals(expect, fs.getFileStatus(p).getModificationTime());

  ManualEnvironmentEdge mockEnv = new ManualEnvironmentEdge();
  mockEnv.setValue(expect);
  EnvironmentEdgeManager.injectEdge(mockEnv);
  try {
    String dstFile = UUID.randomUUID().toString();
    Path dst = new Path(testDir , dstFile);

    assertTrue(FSUtils.renameAndSetModifyTime(fs, p, dst));
    assertFalse("The moved file should not be present", FSUtils.isExists(fs, p));
    assertTrue("The dst file should be present", FSUtils.isExists(fs, dst));

    assertEquals(expect, fs.getFileStatus(dst).getModificationTime());
    cluster.shutdown();
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example 59
Project: ditb   File: TableSnapshotInputFormatTestBase.java   Source Code and License 5 votes vote down vote up
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
    int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
  setupCluster();
  try {
    Path tableDir = util.getRandomDir();
    TableName tableName = TableName.valueOf("testWithMapReduce");
    testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
      expectedNumSplits, shutdownCluster);
  } finally {
    tearDownCluster();
  }
}
 
Example 60
Project: ditb   File: TestFileLink.java   Source Code and License 4 votes vote down vote up
/**
 * Test that link is still readable even when the current file gets deleted.
 *
 * NOTE: This test is valid only on HDFS.
 * When a file is deleted from a local file-system, it is simply 'unlinked'.
 * The inode, which contains the file's data, is not deleted until all
 * processes have finished with it.
 * In HDFS when the request exceed the cached block locations,
 * a query to the namenode is performed, using the filename,
 * and the deleted file doesn't exists anymore (FileNotFoundException).
 */
@Test
public void testHDFSLinkReadDuringDelete() throws Exception {
  HBaseTestingUtility testUtil = new HBaseTestingUtility();
  Configuration conf = testUtil.getConfiguration();
  conf.setInt("dfs.blocksize", 1024 * 1024);
  conf.setInt("dfs.client.read.prefetch.size", 2 * 1024 * 1024);

  testUtil.startMiniDFSCluster(1);
  MiniDFSCluster cluster = testUtil.getDFSCluster();
  FileSystem fs = cluster.getFileSystem();
  assertEquals("hdfs", fs.getUri().getScheme());

  try {
    List<Path> files = new ArrayList<Path>();
    for (int i = 0; i < 3; i++) {
      Path path = new Path(String.format("test-data-%d", i));
      writeSomeData(fs, path, 1 << 20, (byte)i);
      files.add(path);
    }

    FileLink link = new FileLink(files);
    FSDataInputStream in = link.open(fs);
    try {
      byte[] data = new byte[8192];
      int n;

      // Switch to file 1
      n = in.read(data);
      dataVerify(data, n, (byte)0);
      fs.delete(files.get(0), true);
      skipBuffer(in, (byte)0);

      // Switch to file 2
      n = in.read(data);
      dataVerify(data, n, (byte)1);
      fs.delete(files.get(1), true);
      skipBuffer(in, (byte)1);

      // Switch to file 3
      n = in.read(data);
      dataVerify(data, n, (byte)2);
      fs.delete(files.get(2), true);
      skipBuffer(in, (byte)2);

      // No more files available
      try {
        n = in.read(data);
        assert(n <= 0);
      } catch (FileNotFoundException e) {
        assertTrue(true);
      }
    } finally {
      in.close();
    }
  } finally {
    testUtil.shutdownMiniCluster();
  }
}