org.apache.hadoop.hbase.Stoppable Java Examples

The following examples show how to use org.apache.hadoop.hbase.Stoppable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestParalleIndexWriter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, HTableInterface> emptyMap());
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  Abortable mockAbort = Mockito.mock(Abortable.class);
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockAbort, mockStop, 1);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockAbort, mockStop);
}
 
Example #2
Source File: HFileCleaner.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * For creating customized HFileCleaner.
 * @param name name of the chore being run
 * @param period the period of time to sleep between each run
 * @param stopper the stopper
 * @param conf configuration to use
 * @param fs handle to the FS
 * @param directory directory to be cleaned
 * @param confKey configuration key for the classes to instantiate
 * @param pool the thread pool used to scan directories
 * @param params params could be used in subclass of BaseHFileCleanerDelegate
 */
public HFileCleaner(String name, int period, Stoppable stopper, Configuration conf, FileSystem fs,
  Path directory, String confKey, DirScanPool pool, Map<String, Object> params) {
  super(name, period, stopper, conf, fs, directory, confKey, pool, params);
  throttlePoint =
    conf.getInt(HFILE_DELETE_THROTTLE_THRESHOLD, DEFAULT_HFILE_DELETE_THROTTLE_THRESHOLD);
  largeQueueInitSize =
    conf.getInt(LARGE_HFILE_QUEUE_INIT_SIZE, DEFAULT_LARGE_HFILE_QUEUE_INIT_SIZE);
  smallQueueInitSize =
    conf.getInt(SMALL_HFILE_QUEUE_INIT_SIZE, DEFAULT_SMALL_HFILE_QUEUE_INIT_SIZE);
  largeFileQueue = new StealJobQueue<>(largeQueueInitSize, smallQueueInitSize, COMPARATOR);
  smallFileQueue = largeFileQueue.getStealFromQueue();
  largeFileDeleteThreadNumber =
    conf.getInt(LARGE_HFILE_DELETE_THREAD_NUMBER, DEFAULT_LARGE_HFILE_DELETE_THREAD_NUMBER);
  smallFileDeleteThreadNumber =
    conf.getInt(SMALL_HFILE_DELETE_THREAD_NUMBER, DEFAULT_SMALL_HFILE_DELETE_THREAD_NUMBER);
  cleanerThreadTimeoutMsec =
    conf.getLong(HFILE_DELETE_THREAD_TIMEOUT_MSEC, DEFAULT_HFILE_DELETE_THREAD_TIMEOUT_MSEC);
  cleanerThreadCheckIntervalMsec = conf.getLong(HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC,
    DEFAULT_HFILE_DELETE_THREAD_CHECK_INTERVAL_MSEC);
  startHFileDeleteThreads();
}
 
Example #3
Source File: TestParalleWriterIndexCommitter.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, HTableInterface> emptyMap());
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter();
  Abortable mockAbort = Mockito.mock(Abortable.class);
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockAbort, mockStop, 1);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockAbort, mockStop);
}
 
Example #4
Source File: TestParalleWriterIndexCommitter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, Table> emptyMap());
  TrackingParallelWriterIndexCommitter writer = new TrackingParallelWriterIndexCommitter(VersionInfo.getVersion());
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  RegionCoprocessorEnvironment e =Mockito.mock(RegionCoprocessorEnvironment.class);
  Configuration conf =new Configuration();
  Mockito.when(e.getConfiguration()).thenReturn(conf);
  Mockito.when(e.getSharedData()).thenReturn(new ConcurrentHashMap<String,Object>());
  // create a simple writer
  writer.setup(factory, exec, mockStop, e);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockStop);
}
 
Example #5
Source File: TestMasterRegionCompaction.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
protected void postSetUp() throws IOException {
  Configuration conf = htu.getConfiguration();
  conf.setLong(TimeToLiveMasterLocalStoreHFileCleaner.TTL_CONF_KEY, 5000);
  Path testDir = htu.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(conf);
  Path globalArchivePath = HFileArchiveUtil.getArchivePath(conf);
  hfileCleaner = new HFileCleaner(500, new Stoppable() {

    private volatile boolean stopped = false;

    @Override
    public void stop(String why) {
      stopped = true;
    }

    @Override
    public boolean isStopped() {
      return stopped;
    }
  }, conf, fs, globalArchivePath, cleanerPool);
  choreService.scheduleChore(hfileCleaner);
}
 
Example #6
Source File: TestParalleIndexWriter.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, HTableInterface> emptyMap());
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter();
  Abortable mockAbort = Mockito.mock(Abortable.class);
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockAbort, mockStop, 1);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockAbort, mockStop);
}
 
Example #7
Source File: TestCleanerChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testStoppedCleanerDoesNotDeleteFiles() throws Exception {
  Stoppable stop = new StoppableImplementation();
  Configuration conf = UTIL.getConfiguration();
  Path testDir = UTIL.getDataTestDir();
  FileSystem fs = UTIL.getTestFileSystem();
  String confKey = "hbase.test.cleaner.delegates";
  conf.set(confKey, AlwaysDelete.class.getName());

  AllValidPaths chore =
    new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);

  // also create a file in the top level directory
  Path topFile = new Path(testDir, "topFile");
  fs.create(topFile).close();
  assertTrue("Test file didn't get created.", fs.exists(topFile));

  // stop the chore
  stop.stop("testing stop");

  // run the chore
  chore.chore();

  // test that the file still exists
  assertTrue("File got deleted while chore was stopped", fs.exists(topFile));
}
 
Example #8
Source File: TestParalleWriterIndexCommitter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectlyCleansUpResources() throws Exception{
  ExecutorService exec = Executors.newFixedThreadPool(1);
  FakeTableFactory factory = new FakeTableFactory(
      Collections.<ImmutableBytesPtr, HTableInterface> emptyMap());
  ParallelWriterIndexCommitter writer = new ParallelWriterIndexCommitter(VersionInfo.getVersion());
  Abortable mockAbort = Mockito.mock(Abortable.class);
  Stoppable mockStop = Mockito.mock(Stoppable.class);
  // create a simple writer
  writer.setup(factory, exec, mockAbort, mockStop, 1);
  // stop the writer
  writer.stop(this.test.getTableNameString() + " finished");
  assertTrue("Factory didn't get shutdown after writer#stop!", factory.shutdown);
  assertTrue("ExectorService isn't terminated after writer#stop!", exec.isShutdown());
  Mockito.verifyZeroInteractions(mockAbort, mockStop);
}
 
Example #9
Source File: TestServerNonceManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testStopWaiting() throws Exception {
  final ServerNonceManager nm = createManager();
  nm.setConflictWaitIterationMs(1);
  Stoppable stoppingStoppable = createStoppable();
  Mockito.when(stoppingStoppable.isStopped()).thenAnswer(new Answer<Boolean>() {
    AtomicInteger answer = new AtomicInteger(3);
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      return 0 < answer.decrementAndGet();
    }
  });

  nm.startOperation(NO_NONCE, 1, createStoppable());
  TestRunnable tr = new TestRunnable(nm, 1, null, stoppingStoppable);
  Thread t = tr.start();
  waitForThreadToBlockOrExit(t);
  // thread must eventually throw
  t.join();
  tr.propagateError();
}
 
Example #10
Source File: TestSnapshotCleanerChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSnapshotCleanerWithSomeTtlExpired() throws IOException {
  snapshotManager = Mockito.mock(SnapshotManager.class);
  Stoppable stopper = new StoppableImplementation();
  Configuration conf = getSnapshotCleanerConf();
  SnapshotCleanerChore snapshotCleanerChore =
          new SnapshotCleanerChore(stopper, conf, snapshotManager);
  List<SnapshotProtos.SnapshotDescription> snapshotDescriptionList = new ArrayList<>();
  snapshotDescriptionList.add(getSnapshotDescription(10, "snapshot01", "table01", 1));
  snapshotDescriptionList.add(getSnapshotDescription(5, "snapshot02", "table02", 2));
  snapshotDescriptionList.add(getSnapshotDescription(30, "snapshot01", "table01",
          EnvironmentEdgeManager.currentTime()));
  snapshotDescriptionList.add(getSnapshotDescription(0, "snapshot02", "table02",
          EnvironmentEdgeManager.currentTime()));
  snapshotDescriptionList.add(getSnapshotDescription(40, "snapshot03", "table03",
          EnvironmentEdgeManager.currentTime()));
  Mockito.when(snapshotManager.getCompletedSnapshots()).thenReturn(snapshotDescriptionList);
  try {
    LOG.info("5 Snapshots are completed. TTL is expired for 2 them. Going to delete them");
    snapshotCleanerChore.chore();
  } finally {
    stopper.stop("Stopping Test Stopper");
  }
  Mockito.verify(snapshotManager, Mockito.times(2)).deleteSnapshot(Mockito.any());
}
 
Example #11
Source File: TestSnapshotCleanerChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSnapshotCleanerWithReadIOE() throws IOException {
  snapshotManager = Mockito.mock(SnapshotManager.class);
  Stoppable stopper = new StoppableImplementation();
  Configuration conf = new HBaseTestingUtility().getConfiguration();
  SnapshotCleanerChore snapshotCleanerChore =
          new SnapshotCleanerChore(stopper, conf, snapshotManager);
  Mockito.when(snapshotManager.getCompletedSnapshots()).thenThrow(IOException.class);
  try {
    LOG.info("While getting completed Snapshots, IOException would occur. Hence, No Snapshot"
            + " should be deleted");
    snapshotCleanerChore.chore();
  } finally {
    stopper.stop("Stopping Test Stopper");
  }
  Mockito.verify(snapshotManager, Mockito.times(0)).deleteSnapshot(Mockito.any());
}
 
Example #12
Source File: TestMasterRegionWALCleaner.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
protected void postSetUp() throws IOException {
  Configuration conf = htu.getConfiguration();
  conf.setLong(TimeToLiveMasterLocalStoreWALCleaner.TTL_CONF_KEY, TTL_MS);
  Path testDir = htu.getDataTestDir();
  FileSystem fs = testDir.getFileSystem(conf);
  globalWALArchiveDir = new Path(testDir, HConstants.HREGION_OLDLOGDIR_NAME);
  logCleaner = new LogCleaner(1000, new Stoppable() {

    private volatile boolean stopped = false;

    @Override
    public void stop(String why) {
      stopped = true;
    }

    @Override
    public boolean isStopped() {
      return stopped;
    }
  }, conf, fs, globalWALArchiveDir, cleanerPool);
  choreService.scheduleChore(logCleaner);
}
 
Example #13
Source File: TestSnapshotCleanerChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSnapshotChoreWithTtlOutOfRange() throws IOException {
  snapshotManager = Mockito.mock(SnapshotManager.class);
  Stoppable stopper = new StoppableImplementation();
  Configuration conf = getSnapshotCleanerConf();
  List<SnapshotProtos.SnapshotDescription> snapshotDescriptionList = new ArrayList<>();
  snapshotDescriptionList.add(getSnapshotDescription(Long.MAX_VALUE, "snapshot01", "table01", 1));
  snapshotDescriptionList.add(getSnapshotDescription(5, "snapshot02", "table02", 2));
  Mockito.when(snapshotManager.getCompletedSnapshots()).thenReturn(snapshotDescriptionList);
  SnapshotCleanerChore snapshotCleanerChore =
          new SnapshotCleanerChore(stopper, conf, snapshotManager);
  try {
    LOG.info("Snapshot Chore is disabled. No cleanup performed for Expired Snapshots");
    snapshotCleanerChore.chore();
  } finally {
    stopper.stop("Stopping Test Stopper");
  }
  Mockito.verify(snapshotManager, Mockito.times(1)).getCompletedSnapshots();
}
 
Example #14
Source File: TestEndToEndSplitTransaction.java    From hbase with Apache License 2.0 5 votes vote down vote up
RegionChecker(Configuration conf, Stoppable stopper, TableName tableName) throws IOException {
  super("RegionChecker", stopper, 100);
  this.conf = conf;
  this.tableName = tableName;

  this.connection = ConnectionFactory.createConnection(conf);
}
 
Example #15
Source File: TestServerNonceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testCleanup() throws Exception {
  ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
  EnvironmentEdgeManager.injectEdge(edge);
  try {
    ServerNonceManager nm = createManager(6);
    ScheduledChore cleanup = nm.createCleanupScheduledChore(Mockito.mock(Stoppable.class));
    edge.setValue(1);
    assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable()));
    assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable()));
    assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable()));
    edge.setValue(2);
    nm.endOperation(NO_NONCE, 1, true);
    edge.setValue(4);
    nm.endOperation(NO_NONCE, 2, true);
    edge.setValue(9);
    cleanup.choreForTesting();
    // Nonce 1 has been cleaned up.
    assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable()));
    // Nonce 2 has not been cleaned up.
    assertFalse(nm.startOperation(NO_NONCE, 2, createStoppable()));
    // Nonce 3 was active and active ops should never be cleaned up; try to end and start.
    nm.endOperation(NO_NONCE, 3, false);
    assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable()));
    edge.setValue(11);
    cleanup.choreForTesting();
    // Now, nonce 2 has been cleaned up.
    assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable()));
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example #16
Source File: TestCleanerChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test to make sure that we don't attempt to ask the delegate whether or not we should preserve a
 * directory.
 * @throws Exception on failure
 */
@Test
public void testDoesNotCheckDirectories() throws Exception {
  Stoppable stop = new StoppableImplementation();
  Configuration conf = UTIL.getConfiguration();
  Path testDir = UTIL.getDataTestDir();
  FileSystem fs = UTIL.getTestFileSystem();
  String confKey = "hbase.test.cleaner.delegates";
  conf.set(confKey, AlwaysDelete.class.getName());

  AllValidPaths chore =
    new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);
  // spy on the delegate to ensure that we don't check for directories
  AlwaysDelete delegate = (AlwaysDelete) chore.cleanersChain.get(0);
  AlwaysDelete spy = Mockito.spy(delegate);
  chore.cleanersChain.set(0, spy);

  // create the directory layout in the directory to clean
  Path parent = new Path(testDir, "parent");
  Path file = new Path(parent, "someFile");
  fs.mkdirs(parent);
  assertTrue("Test parent didn't get created.", fs.exists(parent));
  // touch a new file
  fs.create(file).close();
  assertTrue("Test file didn't get created.", fs.exists(file));

  FileStatus fStat = fs.getFileStatus(parent);
  chore.chore();
  // make sure we never checked the directory
  Mockito.verify(spy, Mockito.never()).isFileDeletable(fStat);
  Mockito.reset(spy);
}
 
Example #17
Source File: TestServerNonceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testWalNonces() throws Exception {
  ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
  EnvironmentEdgeManager.injectEdge(edge);
  try {
    ServerNonceManager nm = createManager(6);
    ScheduledChore cleanup = nm.createCleanupScheduledChore(Mockito.mock(Stoppable.class));
    // Add nonces from WAL, including dups.
    edge.setValue(12);
    nm.reportOperationFromWal(NO_NONCE, 1, 8);
    nm.reportOperationFromWal(NO_NONCE, 2, 2);
    nm.reportOperationFromWal(NO_NONCE, 3, 5);
    nm.reportOperationFromWal(NO_NONCE, 3, 6);
    // WAL nonces should prevent cross-server conflicts.
    assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable()));
    // Make sure we ignore very old nonces, but not borderline old nonces.
    assertTrue(nm.startOperation(NO_NONCE, 2, createStoppable()));
    assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable()));
    // Make sure grace period is counted from recovery time.
    edge.setValue(17);
    cleanup.choreForTesting();
    assertFalse(nm.startOperation(NO_NONCE, 1, createStoppable()));
    assertFalse(nm.startOperation(NO_NONCE, 3, createStoppable()));
    edge.setValue(19);
    cleanup.choreForTesting();
    assertTrue(nm.startOperation(NO_NONCE, 1, createStoppable()));
    assertTrue(nm.startOperation(NO_NONCE, 3, createStoppable()));
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example #18
Source File: HRegionServer.java    From hbase with Apache License 2.0 5 votes vote down vote up
CompactionChecker(final HRegionServer h, final int sleepTime, final Stoppable stopper) {
  super("CompactionChecker", stopper, sleepTime);
  this.instance = h;
  LOG.info(this.getName() + " runs every " + Duration.ofMillis(sleepTime));

  /* MajorCompactPriority is configurable.
   * If not set, the compaction will use default priority.
   */
  this.majorCompactPriority = this.instance.conf.
      getInt("hbase.regionserver.compactionChecker.majorCompactPriority",
          DEFAULT_PRIORITY);
}
 
Example #19
Source File: HRegionServer.java    From hbase with Apache License 2.0 5 votes vote down vote up
static MovedRegionsCleaner create(HRegionServer rs){
  Stoppable stoppable = new Stoppable() {
    private volatile boolean isStopped = false;
    @Override public void stop(String why) { isStopped = true;}
    @Override public boolean isStopped() {return isStopped;}
  };

  return new MovedRegionsCleaner(rs, stoppable);
}
 
Example #20
Source File: LogCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @param period the period of time to sleep between each run
 * @param stopper the stopper
 * @param conf configuration to use
 * @param fs handle to the FS
 * @param oldLogDir the path to the archived logs
 * @param pool the thread pool used to scan directories
 */
public LogCleaner(final int period, final Stoppable stopper, Configuration conf, FileSystem fs,
  Path oldLogDir, DirScanPool pool) {
  super("LogsCleaner", period, stopper, conf, fs, oldLogDir, HBASE_MASTER_LOGCLEANER_PLUGINS,
    pool);
  this.pendingDelete = new LinkedBlockingQueue<>();
  int size = conf.getInt(OLD_WALS_CLEANER_THREAD_SIZE, DEFAULT_OLD_WALS_CLEANER_THREAD_SIZE);
  this.oldWALsCleaner = createOldWalsCleaner(size);
  this.cleanerThreadTimeoutMsec = conf.getLong(OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC,
    DEFAULT_OLD_WALS_CLEANER_THREAD_TIMEOUT_MSEC);
}
 
Example #21
Source File: TestCleanerChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeletesEmptyDirectories() throws Exception {
  Stoppable stop = new StoppableImplementation();
  Configuration conf = UTIL.getConfiguration();
  Path testDir = UTIL.getDataTestDir();
  FileSystem fs = UTIL.getTestFileSystem();
  String confKey = "hbase.test.cleaner.delegates";
  conf.set(confKey, AlwaysDelete.class.getName());

  AllValidPaths chore =
    new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);

  // create the directory layout in the directory to clean
  Path parent = new Path(testDir, "parent");
  Path child = new Path(parent, "child");
  Path emptyChild = new Path(parent, "emptyChild");
  Path file = new Path(child, "someFile");
  fs.mkdirs(child);
  fs.mkdirs(emptyChild);
  // touch a new file
  fs.create(file).close();
  // also create a file in the top level directory
  Path topFile = new Path(testDir, "topFile");
  fs.create(topFile).close();
  assertTrue("Test file didn't get created.", fs.exists(file));
  assertTrue("Test file didn't get created.", fs.exists(topFile));

  // run the chore
  chore.chore();

  // verify all the files got deleted
  assertFalse("File didn't get deleted", fs.exists(topFile));
  assertFalse("File didn't get deleted", fs.exists(file));
  assertFalse("Empty directory didn't get deleted", fs.exists(child));
  assertFalse("Empty directory didn't get deleted", fs.exists(parent));
}
 
Example #22
Source File: ShutdownHook.java    From hbase with Apache License 2.0 5 votes vote down vote up
ShutdownHookThread(final Configuration conf, final Stoppable stop,
    final Thread threadToJoin, final Runnable fsShutdownHook) {
  super("Shutdownhook:" + threadToJoin.getName());
  this.stop = stop;
  this.threadToJoin = threadToJoin;
  this.conf = conf;
  this.fsShutdownHook = fsShutdownHook;
}
 
Example #23
Source File: ServerNonceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a scheduled chore that is used to clean up old nonces.
 * @param stoppable Stoppable for the chore.
 * @return ScheduledChore; the scheduled chore is not started.
 */
public ScheduledChore createCleanupScheduledChore(Stoppable stoppable) {
  // By default, it will run every 6 minutes (30 / 5).
  return new ScheduledChore("nonceCleaner", stoppable, deleteNonceGracePeriod / 5) {
    @Override
    protected void chore() {
      cleanUpOldNonces();
    }
  };
}
 
Example #24
Source File: CompactedHFilesDischarger.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @param period the period of time to sleep between each run
 * @param stopper the stopper
 * @param regionServerServices the region server that starts this chore
 */
public CompactedHFilesDischarger(final int period, final Stoppable stopper,
    final RegionServerServices regionServerServices) {
  // Need to add the config classes
  super("CompactedHFilesCleaner", stopper, period);
  this.regionServerServices = regionServerServices;
}
 
Example #25
Source File: CompactedHFilesDischarger.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @param period the period of time to sleep between each run
 * @param stopper the stopper
 * @param regionServerServices the region server that starts this chore
 * @param useExecutor true if to use the region server's executor service, false otherwise
 */
@VisibleForTesting
public CompactedHFilesDischarger(final int period, final Stoppable stopper,
    final RegionServerServices regionServerServices, boolean useExecutor) {
  // Need to add the config classes
  this(period, stopper, regionServerServices);
  this.useExecutor = useExecutor;
}
 
Example #26
Source File: StoreFailuresInCachePolicy.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void setup(Stoppable parent, RegionCoprocessorEnvironment env) {
  this.region = env.getRegion();
  this.delegate = new KillServerOnFailurePolicy();
  this.delegate.setup(parent, env);

}
 
Example #27
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithoutStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.unset("hbase.regions.recovery.store.file.ref.count");
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that by default the feature is turned off so no regions
  // should be reopened
  Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());

  // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
  Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(0))
    .getRegionInfo(Mockito.any());
}
 
Example #28
Source File: TestSnapshotCleanerChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testSnapshotCleanerWithoutAnyCompletedSnapshot() throws IOException {
  snapshotManager = Mockito.mock(SnapshotManager.class);
  Stoppable stopper = new StoppableImplementation();
  Configuration conf = getSnapshotCleanerConf();
  SnapshotCleanerChore snapshotCleanerChore =
          new SnapshotCleanerChore(stopper, conf, snapshotManager);
  try {
    snapshotCleanerChore.chore();
  } finally {
    stopper.stop("Stopping Test Stopper");
  }
  Mockito.verify(snapshotManager, Mockito.times(0)).deleteSnapshot(Mockito.any());
}
 
Example #29
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithLessThreshold() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of only 1 table
  Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen only 1 region with refCount > 400
  Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(1))
    .getRegionInfo(Mockito.any());
}
 
Example #30
Source File: TestCleanerChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testSavesFilesOnRequest() throws Exception {
  Stoppable stop = new StoppableImplementation();
  Configuration conf = UTIL.getConfiguration();
  Path testDir = UTIL.getDataTestDir();
  FileSystem fs = UTIL.getTestFileSystem();
  String confKey = "hbase.test.cleaner.delegates";
  conf.set(confKey, NeverDelete.class.getName());

  AllValidPaths chore =
    new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey, POOL);

  // create the directory layout in the directory to clean
  Path parent = new Path(testDir, "parent");
  Path file = new Path(parent, "someFile");
  fs.mkdirs(parent);
  // touch a new file
  fs.create(file).close();
  assertTrue("Test file didn't get created.", fs.exists(file));

  // run the chore
  chore.chore();

  // verify all the files were preserved
  assertTrue("File shouldn't have been deleted", fs.exists(file));
  assertTrue("directory shouldn't have been deleted", fs.exists(parent));
}