Java Code Examples for org.apache.hadoop.hbase.ChoreService

The following examples show how to use org.apache.hadoop.hbase.ChoreService. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: HMaster.java    License: Apache License 2.0 6 votes vote down vote up
private void stopChores() {
  ChoreService choreService = getChoreService();
  if (choreService != null) {
    choreService.cancelChore(this.mobFileCleanerChore);
    choreService.cancelChore(this.mobFileCompactionChore);
    choreService.cancelChore(this.balancerChore);
    choreService.cancelChore(this.normalizerChore);
    choreService.cancelChore(this.clusterStatusChore);
    choreService.cancelChore(this.catalogJanitorChore);
    choreService.cancelChore(this.clusterStatusPublisherChore);
    choreService.cancelChore(this.snapshotQuotaChore);
    choreService.cancelChore(this.logCleaner);
    choreService.cancelChore(this.hfileCleaner);
    choreService.cancelChore(this.replicationBarrierCleaner);
    choreService.cancelChore(this.snapshotCleanerChore);
    choreService.cancelChore(this.hbckChore);
    choreService.cancelChore(this.regionsRecoveryChore);
  }
}
 
Example 2
Source Project: hbase   Source File: TestHeapMemoryManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWhenSizeGivenByHeapTunerGoesOutsideRange() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
      HeapMemoryTuner.class);
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  CustomHeapMemoryTuner.memstoreSize = 0.78f;
  CustomHeapMemoryTuner.blockCacheSize = 0.02f;
  Thread.sleep(1500); // Allow the tuner to run once and do necessary memory up
  // Even if the tuner says to set the memstore to 78%, HBase makes it as 70% as that is the
  // upper bound. Same with block cache as 10% is the lower bound.
  assertHeapSpace(0.7f, memStoreFlusher.memstoreSize);
  assertHeapSpace(0.1f, blockCache.maxSize);
}
 
Example 3
Source Project: hbase   Source File: TestHeapMemoryManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWhenCombinedHeapSizesFromTunerGoesOutSideMaxLimit() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
      HeapMemoryTuner.class);
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
  long oldMemstoreSize = memStoreFlusher.memstoreSize;
  long oldBlockCacheSize = blockCache.maxSize;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  CustomHeapMemoryTuner.memstoreSize = 0.7f;
  CustomHeapMemoryTuner.blockCacheSize = 0.3f;
  // Allow the tuner to run once and do necessary memory up
  Thread.sleep(1500);
  assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
  assertEquals(oldBlockCacheSize, blockCache.maxSize);
}
 
Example 4
Source Project: hbase   Source File: SplitLogManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Its OK to construct this object even when region-servers are not online. It does lookup the
 * orphan tasks in coordination engine but it doesn't block waiting for them to be done.
 * @param master the master services
 * @param conf the HBase configuration
 * @throws IOException
 */
public SplitLogManager(MasterServices master, Configuration conf)
    throws IOException {
  this.server = master;
  this.conf = conf;
  // If no CoordinatedStateManager, skip registering as a chore service (The
  // CoordinatedStateManager is non-null if we are running the ZK-based distributed WAL
  // splitting. It is null if we are configured to use procedure-based distributed WAL
  // splitting.
  if (server.getCoordinatedStateManager() != null) {
    this.choreService =
      new ChoreService(master.getServerName().toShortString() + ".splitLogManager.");
    SplitLogManagerCoordination coordination = getSplitLogManagerCoordination();
    Set<String> failedDeletions = Collections.synchronizedSet(new HashSet<String>());
    SplitLogManagerDetails details = new SplitLogManagerDetails(tasks, master, failedDeletions);
    coordination.setDetails(details);
    coordination.init();
    this.unassignedTimeout =
      conf.getInt("hbase.splitlog.manager.unassigned.timeout", DEFAULT_UNASSIGNED_TIMEOUT);
    this.timeoutMonitor =
      new TimeoutMonitor(conf.getInt("hbase.splitlog.manager.timeoutmonitor.period", 1000),
        master);
    this.choreService.scheduleChore(timeoutMonitor);
  } else {
    this.choreService = null;
    this.timeoutMonitor = null;
  }
}
 
Example 5
Source Project: hbase   Source File: RegionsRecoveryConfigManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onConfigurationChange(Configuration conf) {
  final int newMaxStoreFileRefCount = getMaxStoreFileRefCount(conf);
  final int newRegionsRecoveryInterval = getRegionsRecoveryChoreInterval(conf);

  if (prevMaxStoreFileRefCount == newMaxStoreFileRefCount
      && prevRegionsRecoveryInterval == newRegionsRecoveryInterval) {
    // no need to re-schedule the chore with updated config
    // as there is no change in desired configs
    return;
  }

  LOG.info("Config Reload for RegionsRecovery Chore. prevMaxStoreFileRefCount: {}," +
      " newMaxStoreFileRefCount: {}, prevRegionsRecoveryInterval: {}, " +
      "newRegionsRecoveryInterval: {}", prevMaxStoreFileRefCount, newMaxStoreFileRefCount,
    prevRegionsRecoveryInterval, newRegionsRecoveryInterval);

  RegionsRecoveryChore regionsRecoveryChore = new RegionsRecoveryChore(this.hMaster,
    conf, this.hMaster);
  ChoreService choreService = this.hMaster.getChoreService();

  // Regions Reopen based on very high storeFileRefCount is considered enabled
  // only if hbase.regions.recovery.store.file.ref.count has value > 0

  synchronized (this) {
    if (newMaxStoreFileRefCount > 0) {
      // reschedule the chore
      // provide mayInterruptIfRunning - false to take care of completion
      // of in progress task if any
      choreService.cancelChore(regionsRecoveryChore, false);
      choreService.scheduleChore(regionsRecoveryChore);
    } else {
      choreService.cancelChore(regionsRecoveryChore, false);
    }
    this.prevMaxStoreFileRefCount = newMaxStoreFileRefCount;
    this.prevRegionsRecoveryInterval = newRegionsRecoveryInterval;
  }
}
 
Example 6
Source Project: hbase   Source File: HeapMemoryManager.java    License: Apache License 2.0 5 votes vote down vote up
public void start(ChoreService service) {
  LOG.info("Starting, tuneOn={}", this.tunerOn);
  this.heapMemTunerChore = new HeapMemoryTunerChore();
  service.scheduleChore(heapMemTunerChore);
  if (tunerOn) {
    // Register HeapMemoryTuner as a memstore flush listener
    memStoreFlusher.registerFlushRequestListener(heapMemTunerChore);
  }
}
 
Example 7
Source Project: hbase   Source File: TestZooKeeperTableArchiveClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param cleaner the cleaner to use
 */
private void runCleaner(HFileCleaner cleaner, CountDownLatch finished, Stoppable stop)
    throws InterruptedException {
  final ChoreService choreService = new ChoreService("CLEANER_SERVER_NAME");
  // run the cleaner
  choreService.scheduleChore(cleaner);
  // wait for the cleaner to check all the files
  finished.await();
  // stop the cleaner
  stop.stop("");
}
 
Example 8
Source Project: hbase   Source File: TestEndToEndSplitTransaction.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the client sees meta table changes as atomic during splits
 */
@Test
public void testFromClientSideWhileSplitting() throws Throwable {
  LOG.info("Starting testFromClientSideWhileSplitting");
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final byte[] FAMILY = Bytes.toBytes("family");

  // SplitTransaction will update the meta table by offlining the parent region, and adding info
  // for daughters.
  Table table = TEST_UTIL.createTable(tableName, FAMILY);

  Stoppable stopper = new StoppableImplementation();
  RegionSplitter regionSplitter = new RegionSplitter(table);
  RegionChecker regionChecker = new RegionChecker(CONF, stopper, tableName);
  final ChoreService choreService = new ChoreService("TEST_SERVER");

  choreService.scheduleChore(regionChecker);
  regionSplitter.start();

  // wait until the splitter is finished
  regionSplitter.join();
  stopper.stop(null);

  if (regionChecker.ex != null) {
    throw new AssertionError("regionChecker", regionChecker.ex);
  }

  if (regionSplitter.ex != null) {
    throw new AssertionError("regionSplitter", regionSplitter.ex);
  }

  // one final check
  regionChecker.verify();
}
 
Example 9
Source Project: hbase   Source File: TestHeapMemoryManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWhenClusterIsWriteHeavyWithEmptyMemstore() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
  MemstoreFlusherStub memStoreFlusher =
      new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  // Empty block cache and memstore
  blockCache.setTestBlockSize(0);
  regionServerAccounting.setTestMemstoreSize(0);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  // Let the system start with default values for memstore heap and block cache size.
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), regionServerAccounting);
  long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  long oldBlockCacheSize = blockCache.maxSize;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_HIGHER_MARK;
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.flushType = FlushType.ABOVE_ONHEAP_LOWER_MARK;
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  // Allow the tuner to run once and do necessary memory up
  Thread.sleep(1500);
  // No changes should be made by tuner as we already have lot of empty space
  assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertEquals(oldBlockCacheSize, blockCache.maxSize);
}
 
Example 10
Source Project: hbase   Source File: TestHeapMemoryManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWhenClusterIsReadHeavyWithEmptyBlockCache() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
  MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  // Empty block cache and memstore
  blockCache.setTestBlockSize(0);
  regionServerAccounting.setTestMemstoreSize(0);
  // Let the system start with default values for memstore heap and block cache size.
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), regionServerAccounting);
  long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  long oldBlockCacheSize = blockCache.maxSize;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  blockCache.evictBlock(null);
  blockCache.evictBlock(null);
  blockCache.evictBlock(null);
  // Allow the tuner to run once and do necessary memory up
  Thread.sleep(1500);
  // No changes should be made by tuner as we already have lot of empty space
  assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertEquals(oldBlockCacheSize, blockCache.maxSize);
}
 
Example 11
Source Project: hbase   Source File: TestHeapMemoryManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWhenClusterIsWriteHeavyWithOffheapMemstore() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  RegionServerAccountingStub regionServerAccounting = new RegionServerAccountingStub(conf);
  MemstoreFlusherStub memStoreFlusher =
      new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  // Empty block cache and but nearly filled memstore
  blockCache.setTestBlockSize(0);
  regionServerAccounting.setTestMemstoreSize((long) (maxHeapSize * 0.4 * 0.8));
  // Let the system start with default values for memstore heap and block cache size.
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), regionServerAccounting);
  long oldMemstoreHeapSize = memStoreFlusher.memstoreSize;
  long oldBlockCacheSize = blockCache.maxSize;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  // this should not change anything with onheap memstore
  memStoreFlusher.flushType = FlushType.ABOVE_OFFHEAP_HIGHER_MARK;
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  memStoreFlusher.requestFlush(null, FlushLifeCycleTracker.DUMMY);
  // Allow the tuner to run once and do necessary memory up
  Thread.sleep(1500);
  // No changes should be made by tuner as we already have lot of empty space
  assertEquals(oldMemstoreHeapSize, memStoreFlusher.memstoreSize);
  assertEquals(oldBlockCacheSize, blockCache.maxSize);
}
 
Example 12
Source Project: hbase   Source File: TestHeapMemoryManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPluggingInHeapMemoryTuner() throws Exception {
  BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
  MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.4));
  Configuration conf = HBaseConfiguration.create();
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.78f);
  conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f);
  conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.02f);
  conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
  conf.setInt(DefaultHeapMemoryTuner.NUM_PERIODS_TO_IGNORE, 0);
  conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
      HeapMemoryTuner.class);
  // Let the system start with default values for memstore heap and block cache size.
  HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher,
      new RegionServerStub(conf), new RegionServerAccountingStub(conf));
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");
  heapMemoryManager.start(choreService);
  // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner
  CustomHeapMemoryTuner.memstoreSize = 0.78f;
  CustomHeapMemoryTuner.blockCacheSize = 0.02f;
  // Allow the tuner to run once and do necessary memory up
  waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
  assertHeapSpace(0.78f, memStoreFlusher.memstoreSize);// Memstore
  assertHeapSpace(0.02f, blockCache.maxSize);// BlockCache
  // Now we wants to be in read mode. Set bigger memstore size from CustomHeapMemoryTuner
  CustomHeapMemoryTuner.blockCacheSize = 0.75f;
  CustomHeapMemoryTuner.memstoreSize = 0.05f;
  // Allow the tuner to run once and do necessary memory up
  waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize);
  assertHeapSpace(0.75f, blockCache.maxSize);// BlockCache
  assertHeapSpace(0.05f, memStoreFlusher.memstoreSize);// Memstore
}
 
Example 13
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 4 votes vote down vote up
public LocalRegionServerServices(Configuration conf, ServerName serverName) {
  super(conf);
  this.serverName = serverName;
  this.choreService = new ChoreService(getServerName().toString(), true);
}
 
Example 14
Source Project: phoenix-tephra   Source File: TransactionProcessorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return choreService;
}
 
Example 15
Source Project: hbase   Source File: ReplicationSyncUp.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 16
Source Project: hbase   Source File: HRegionServer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return choreService;
}
 
Example 17
Source Project: hbase   Source File: TestReplicationTrackerZKImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 18
Source Project: hbase   Source File: TestReplicationSourceManager.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 19
Source Project: hbase   Source File: TestTokenAuthentication.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 20
@Override
public ChoreService getChoreService() {
  throw new UnsupportedOperationException();
}
 
Example 21
Source Project: hbase   Source File: TestLogsCleaner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 22
Source Project: hbase   Source File: TestHFileLinkCleaner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 23
Source Project: hbase   Source File: TestHFileCleaner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 24
Source Project: hbase   Source File: TestReplicationHFileCleaner.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 25
Source Project: hbase   Source File: MockNoopMasterServices.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 26
Source Project: hbase   Source File: MockRegionServer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 27
Source Project: hbase   Source File: TestActiveMasterManager.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 28
Source Project: hbase   Source File: TestHFileArchiving.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test HFileArchiver.resolveAndArchive() race condition HBASE-7643
 */
@Test
public void testCleaningRace() throws Exception {
  final long TEST_TIME = 20 * 1000;
  final ChoreService choreService = new ChoreService("TEST_SERVER_NAME");

  Configuration conf = UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
  Path rootDir = UTIL.getDataTestDirOnTestFS("testCleaningRace");
  FileSystem fs = UTIL.getTestFileSystem();

  Path archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY);
  Path regionDir = new Path(CommonFSUtils.getTableDir(new Path("./"),
      TableName.valueOf(name.getMethodName())), "abcdef");
  Path familyDir = new Path(regionDir, "cf");

  Path sourceRegionDir = new Path(rootDir, regionDir);
  fs.mkdirs(sourceRegionDir);

  Stoppable stoppable = new StoppableImplementation();

  // The cleaner should be looping without long pauses to reproduce the race condition.
  HFileCleaner cleaner = getHFileCleaner(stoppable, conf, fs, archiveDir);
  assertNotNull("cleaner should not be null", cleaner);
  try {
    choreService.scheduleChore(cleaner);
    // Keep creating/archiving new files while the cleaner is running in the other thread
    long startTime = System.currentTimeMillis();
    for (long fid = 0; (System.currentTimeMillis() - startTime) < TEST_TIME; ++fid) {
      Path file = new Path(familyDir,  String.valueOf(fid));
      Path sourceFile = new Path(rootDir, file);
      Path archiveFile = new Path(archiveDir, file);

      fs.createNewFile(sourceFile);

      try {
        // Try to archive the file
        HFileArchiver.archiveRegion(fs, rootDir,
            sourceRegionDir.getParent(), sourceRegionDir);

        // The archiver succeded, the file is no longer in the original location
        // but it's in the archive location.
        LOG.debug("hfile=" + fid + " should be in the archive");
        assertTrue(fs.exists(archiveFile));
        assertFalse(fs.exists(sourceFile));
      } catch (IOException e) {
        // The archiver is unable to archive the file. Probably HBASE-7643 race condition.
        // in this case, the file should not be archived, and we should have the file
        // in the original location.
        LOG.debug("hfile=" + fid + " should be in the source location");
        assertFalse(fs.exists(archiveFile));
        assertTrue(fs.exists(sourceFile));

        // Avoid to have this file in the next run
        fs.delete(sourceFile, false);
      }
    }
  } finally {
    stoppable.stop("test end");
    cleaner.cancel(true);
    choreService.shutdown();
    fs.delete(rootDir, true);
  }
}
 
Example 29
Source Project: hbase   Source File: MockServer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public ChoreService getChoreService() {
  return null;
}
 
Example 30
Source Project: hbase   Source File: TestCompaction.java    License: Apache License 2.0 4 votes vote down vote up
/** Test compaction priority management and multiple compactions per store (HBASE-8665). */
@Test
public void testCompactionQueuePriorities() throws Exception {
  // Setup a compact/split thread on a mock server.
  final Configuration conf = HBaseConfiguration.create();
  HRegionServer mockServer = mock(HRegionServer.class);
  when(mockServer.isStopped()).thenReturn(false);
  when(mockServer.getConfiguration()).thenReturn(conf);
  when(mockServer.getChoreService()).thenReturn(new ChoreService("test"));
  CompactSplit cst = new CompactSplit(mockServer);
  when(mockServer.getCompactSplitThread()).thenReturn(cst);
  //prevent large compaction thread pool stealing job from small compaction queue.
  cst.shutdownLongCompactions();
  // Set up the region mock that redirects compactions.
  HRegion r = mock(HRegion.class);
  when(
    r.compact(any(), any(), any(), any())).then(new Answer<Boolean>() {
      @Override
      public Boolean answer(InvocationOnMock invocation) throws Throwable {
        invocation.<CompactionContext>getArgument(0).compact(invocation.getArgument(2), null);
        return true;
      }
  });

  // Set up store mocks for 2 "real" stores and the one we use for blocking CST.
  ArrayList<Integer> results = new ArrayList<>();
  StoreMockMaker sm = new StoreMockMaker(results), sm2 = new StoreMockMaker(results);
  HStore store = sm.createStoreMock("store1");
  HStore store2 = sm2.createStoreMock("store2");
  BlockingStoreMockMaker blocker = new BlockingStoreMockMaker();

  // First, block the compaction thread so that we could muck with queue.
  cst.requestSystemCompaction(r, blocker.createStoreMock(1, "b-pri1"), "b-pri1");
  BlockingStoreMockMaker.BlockingCompactionContext currentBlock = blocker.waitForBlocking();

  // Add 4 files to store1, 3 to store2, and queue compactions; pri 3 and 4 respectively.
  for (int i = 0; i < 4; ++i) {
    sm.notCompacting.add(createFile());
  }
  cst.requestSystemCompaction(r, store, "s1-pri3");
  for (int i = 0; i < 3; ++i) {
    sm2.notCompacting.add(createFile());
  }
  cst.requestSystemCompaction(r, store2, "s2-pri4");
  // Now add 2 more files to store1 and queue compaction - pri 1.
  for (int i = 0; i < 2; ++i) {
    sm.notCompacting.add(createFile());
  }
  cst.requestSystemCompaction(r, store, "s1-pri1");
  // Finally add blocking compaction with priority 2.
  cst.requestSystemCompaction(r, blocker.createStoreMock(2, "b-pri2"), "b-pri2");

  // Unblock the blocking compaction; we should run pri1 and become block again in pri2.
  currentBlock.unblock();
  currentBlock = blocker.waitForBlocking();
  // Pri1 should have "compacted" all 6 files.
  assertEquals(1, results.size());
  assertEquals(6, results.get(0).intValue());
  // Add 2 files to store 1 (it has 2 files now).
  for (int i = 0; i < 2; ++i) {
    sm.notCompacting.add(createFile());
  }
  // Now we have pri4 for store 2 in queue, and pri3 for store1; store1's current priority
  // is 5, however, so it must not preempt store 2. Add blocking compaction at the end.
  cst.requestSystemCompaction(r, blocker.createStoreMock(7, "b-pri7"), "b-pri7");
  currentBlock.unblock();
  currentBlock = blocker.waitForBlocking();
  assertEquals(3, results.size());
  assertEquals(3, results.get(1).intValue()); // 3 files should go before 2 files.
  assertEquals(2, results.get(2).intValue());

  currentBlock.unblock();
  cst.interruptIfNecessary();
}