Java Code Examples for org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker

The following examples show how to use org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix-tephra   Source File: TransactionProcessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompact(
    org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, InternalScanner scanner, ScanType scanType,
    org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();
  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }
  // Also make sure to use the same snapshot for the compaction
  InternalScanner s =
      createStoreScanner(c.getEnvironment(), "compaction", snapshot, scanner, scanType);
  if (s != null) {
    return s;
  }
  return scanner;
}
 
Example 2
Source Project: ranger   Source File: RangerAuthorizationCoprocessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends StoreFile> candidates, CompactionLifeCycleTracker request) throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.preCompactSelection()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.preCompactSelection(c, store, candidates, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.preCompactSelection()");
	}
}
 
Example 3
Source Project: ranger   Source File: RangerAuthorizationCoprocessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker, CompactionRequest request) {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postCompactSelection()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postCompactSelection(c, store, selected, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postCompactSelection()");
	}
}
 
Example 4
Source Project: ranger   Source File: RangerAuthorizationCoprocessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,	InternalScanner scanner,
								  ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {

	final InternalScanner ret;

	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.preCompact()");
	}

	try {
		activatePluginClassLoader();
		ret = implRegionObserver.preCompact(c, store, scanner, scanType, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.preCompact()");
	}

	return ret;
}
 
Example 5
Source Project: ranger   Source File: RangerAuthorizationCoprocessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile,
						CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postCompact()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postCompact(c, store, resultFile, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postCompact()");
	}
}
 
Example 6
Source Project: hbase   Source File: CompactSplit.java    License: Apache License 2.0 6 votes vote down vote up
private Optional<CompactionContext> selectCompaction(HRegion region, HStore store, int priority,
    CompactionLifeCycleTracker tracker, CompactionCompleteTracker completeTracker, User user)
    throws IOException {
  // don't even select for compaction if disableCompactions is set to true
  if (!isCompactionsEnabled()) {
    LOG.info(String.format("User has disabled compactions"));
    return Optional.empty();
  }
  Optional<CompactionContext> compaction = store.requestCompaction(priority, tracker, user);
  if (!compaction.isPresent() && region.getRegionInfo() != null) {
    String reason = "Not compacting " + region.getRegionInfo().getRegionNameAsString() +
        " because compaction request was cancelled";
    tracker.notExecuted(store, reason);
    completeTracker.completed(store);
    LOG.debug(reason);
  }
  return compaction;
}
 
Example 7
Source Project: hbase   Source File: TestMajorCompaction.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test for HBASE-5920
 */
@Test
public void testUserMajorCompactionRequest() throws IOException {
  HStore store = r.getStore(COLUMN_FAMILY);
  createStoreFile(r);
  for (int i = 0; i < MAX_FILES_TO_COMPACT + 1; i++) {
    createStoreFile(r);
  }
  store.triggerMajorCompaction();
  CompactionRequestImpl request = store
    .requestCompaction(PRIORITY_USER, CompactionLifeCycleTracker.DUMMY, null).get().getRequest();
  assertNotNull("Expected to receive a compaction request", request);
  assertEquals(
    "User-requested major compaction should always occur, even if there are too many store files",
    true, request.isMajor());
}
 
Example 8
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
                                  InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
                                  CompactionRequest request) throws IOException {
    try {
        BlockingProbe.blockPreCompact();
        if (!(request instanceof SpliceCompactionRequest)) {
            SpliceLogUtils.error(LOG,"Compaction request must be a SpliceCompactionRequest");
            throw new DoNotRetryIOException();
        }
        SpliceCompactionRequest scr = (SpliceCompactionRequest) request;
        // memstoreAware is injected into the request, where the blocking logic lives, and where compaction
        // count will be incremented and decremented.
        scr.setMemstoreAware(memstoreAware);
        HRegion region = (HRegion) c.getEnvironment().getRegion();
        scr.setRegion(region);
        return scanner == null ? DummyScanner.INSTANCE : scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example 9
Source Project: phoenix-omid   Source File: BaseRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c,
                                  Store store,
                                  InternalScanner scanner,
                                  ScanType scanType,
                                  CompactionLifeCycleTracker tracker,
                                  CompactionRequest request) throws IOException {
    return preCompact(c,store,scanner,scanType,request);
}
 
Example 10
Source Project: phoenix-tephra   Source File: TransactionProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  if (cache.getLatestState() != null) {
    options.readAllVersions();
  }
}
 
Example 11
Source Project: phoenix-tephra   Source File: TransactionProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(
    org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, StoreFile resultFile,
    org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  // Persist the compaction state after a successful compaction
  if (compactionState != null) {
    compactionState.persist();
  }
}
 
Example 12
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  requirePermission(c, "compact", getTableName(c.getEnvironment()),
      null, null, Action.ADMIN, Action.CREATE);
  return scanner;
}
 
Example 13
Source Project: hbase   Source File: CompactSplit.java    License: Apache License 2.0 5 votes vote down vote up
private CompactionCompleteTracker getCompleteTracker(CompactionLifeCycleTracker tracker,
    IntSupplier numberOfStores) {
  if (tracker == CompactionLifeCycleTracker.DUMMY) {
    // a simple optimization to avoid creating unnecessary objects as usually we do not care about
    // the life cycle of a compaction.
    return DUMMY_COMPLETE_TRACKER;
  } else {
    return new AggregatingCompleteTracker(tracker, numberOfStores.getAsInt());
  }
}
 
Example 14
Source Project: hbase   Source File: CompactSplit.java    License: Apache License 2.0 5 votes vote down vote up
private void requestCompactionInternal(HRegion region, String why, int priority,
    boolean selectNow, CompactionLifeCycleTracker tracker,
    CompactionCompleteTracker completeTracker, User user) throws IOException {
  // request compaction on all stores
  for (HStore store : region.stores.values()) {
    requestCompactionInternal(region, store, why, priority, selectNow, tracker, completeTracker,
      user);
  }
}
 
Example 15
Source Project: hbase   Source File: CompactSplit.java    License: Apache License 2.0 5 votes vote down vote up
public CompactionRunner(HStore store, HRegion region, CompactionContext compaction,
    CompactionLifeCycleTracker tracker, CompactionCompleteTracker completeTracker,
    ThreadPoolExecutor parent, User user) {
  this.store = store;
  this.region = region;
  this.compaction = compaction;
  this.tracker = tracker;
  this.completeTracker = completeTracker;
  this.queuedPriority =
      compaction != null ? compaction.getRequest().getPriority() : store.getCompactPriority();
  this.parent = parent;
  this.user = user;
  this.time = EnvironmentEdgeManager.currentTime();
}
 
Example 16
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Called prior to selecting the {@link HStoreFile}s for compaction from the list of currently
 * available candidates.
 * <p>Supports Coprocessor 'bypass' -- 'bypass' is how this method indicates that it changed
 * the passed in <code>candidates</code>.
 * @param store The store where compaction is being requested
 * @param candidates The currently available store files
 * @param tracker used to track the life cycle of a compaction
 * @param user the user
 * @throws IOException
 */
public boolean preCompactSelection(final HStore store, final List<HStoreFile> candidates,
    final CompactionLifeCycleTracker tracker, final User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return false;
  }
  boolean bypassable = true;
  return execOperation(new RegionObserverOperationWithoutResult(user, bypassable) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preCompactSelection(this, store, candidates, tracker);
    }
  });
}
 
Example 17
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Called after the {@link HStoreFile}s to be compacted have been selected from the available
 * candidates.
 * @param store The store where compaction is being requested
 * @param selected The store files selected to compact
 * @param tracker used to track the life cycle of a compaction
 * @param request the compaction request
 * @param user the user
 */
public void postCompactSelection(final HStore store, final List<HStoreFile> selected,
    final CompactionLifeCycleTracker tracker, final CompactionRequest request,
    final User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return;
  }
  execOperation(new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postCompactSelection(this, store, selected, tracker, request);
    }
  });
}
 
Example 18
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Called prior to opening store scanner for compaction.
 */
public ScanInfo preCompactScannerOpen(HStore store, ScanType scanType,
    CompactionLifeCycleTracker tracker, CompactionRequest request, User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return store.getScanInfo();
  }
  CustomizedScanInfoBuilder builder = new CustomizedScanInfoBuilder(store.getScanInfo());
  execOperation(new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preCompactScannerOpen(this, store, scanType, builder, tracker, request);
    }
  });
  return builder.build();
}
 
Example 19
Source Project: hbase   Source File: RegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Called after the store compaction has completed.
 * @param store the store being compacted
 * @param resultFile the new store file written during compaction
 * @param tracker used to track the life cycle of a compaction
 * @param request the compaction request
 * @param user the user
 * @throws IOException
 */
public void postCompact(final HStore store, final HStoreFile resultFile,
    final CompactionLifeCycleTracker tracker, final CompactionRequest request, final User user)
    throws IOException {
  execOperation(coprocEnvironments.isEmpty()? null: new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postCompact(this, store, resultFile, tracker, request);
    }
  });
}
 
Example 20
Source Project: hbase   Source File: TestRegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreCompactScannerOpen() throws IOException {
  RegionCoprocessorHost host = new RegionCoprocessorHost(region, rsServices, conf);
  ScanInfo oldScanInfo = getScanInfo();
  HStore store = mock(HStore.class);
  when(store.getScanInfo()).thenReturn(oldScanInfo);
  ScanInfo newScanInfo = host.preCompactScannerOpen(store, ScanType.COMPACT_DROP_DELETES,
    mock(CompactionLifeCycleTracker.class), mock(CompactionRequest.class), mock(User.class));
  verifyScanInfo(newScanInfo);
}
 
Example 21
Source Project: hbase   Source File: TestRegionObserverInterface.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  return new InternalScanner() {

    @Override
    public boolean next(List<Cell> results, ScannerContext scannerContext) throws IOException {
      List<Cell> internalResults = new ArrayList<>();
      boolean hasMore;
      do {
        hasMore = scanner.next(internalResults, scannerContext);
        if (!internalResults.isEmpty()) {
          long row = Bytes.toLong(CellUtil.cloneValue(internalResults.get(0)));
          if (row % 2 == 0) {
            // return this row
            break;
          }
          // clear and continue
          internalResults.clear();
        }
      } while (hasMore);

      if (!internalResults.isEmpty()) {
        results.addAll(internalResults);
      }
      return hasMore;
    }

    @Override
    public void close() throws IOException {
      scanner.close();
    }
  };
}
 
Example 22
Source Project: hbase   Source File: TestCoprocessorInterface.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
    Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  preCompactCalled = true;
  return scanner;
}
 
Example 23
Source Project: hbase   Source File: SimpleRegionObserver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  ctPreCompact.incrementAndGet();
  return scanner;
}
 
Example 24
Source Project: hbase   Source File: TestCompaction.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test no new Compaction requests are generated after calling stop compactions
 */
@Test
public void testStopStartCompaction() throws IOException {
  // setup a compact/split thread on a mock server
  HRegionServer mockServer = Mockito.mock(HRegionServer.class);
  Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf());
  final CompactSplit thread = new CompactSplit(mockServer);
  Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
  // setup a region/store with some files
  HStore store = r.getStore(COLUMN_FAMILY);
  createStoreFile(r);
  for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) {
    createStoreFile(r);
  }
  thread.switchCompaction(false);
  thread.requestCompaction(r, store, "test", Store.PRIORITY_USER,
    CompactionLifeCycleTracker.DUMMY, null);
  assertFalse(thread.isCompactionsEnabled());
  int longCompactions = thread.getLongCompactions().getActiveCount();
  int shortCompactions = thread.getShortCompactions().getActiveCount();
  assertEquals("longCompactions=" + longCompactions + "," +
      "shortCompactions=" + shortCompactions, 0, longCompactions + shortCompactions);
  thread.switchCompaction(true);
  assertTrue(thread.isCompactionsEnabled());
  // Make sure no compactions have run.
  assertEquals(0, thread.getLongCompactions().getCompletedTaskCount() +
      thread.getShortCompactions().getCompletedTaskCount());
  // Request a compaction and make sure it is submitted successfully.
  thread.requestCompaction(r, store, "test", Store.PRIORITY_USER,
      CompactionLifeCycleTracker.DUMMY, null);
  // Wait until the compaction finishes.
  Waiter.waitFor(UTIL.getConfiguration(), 5000,
      (Waiter.Predicate<Exception>) () -> thread.getLongCompactions().getCompletedTaskCount() +
      thread.getShortCompactions().getCompletedTaskCount() == 1);
  // Make sure there are no compactions running.
  assertEquals(0, thread.getLongCompactions().getActiveCount()
      + thread.getShortCompactions().getActiveCount());
}
 
Example 25
Source Project: hbase   Source File: TestCompaction.java    License: Apache License 2.0 5 votes vote down vote up
@Test public void testInterruptingRunningCompactions() throws Exception {
  // setup a compact/split thread on a mock server
  conf.set(CompactionThroughputControllerFactory.HBASE_THROUGHPUT_CONTROLLER_KEY,
      WaitThroughPutController.class.getName());
  HRegionServer mockServer = Mockito.mock(HRegionServer.class);
  Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf());
  CompactSplit thread = new CompactSplit(mockServer);

  Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);

  // setup a region/store with some files
  HStore store = r.getStore(COLUMN_FAMILY);
  int jmax = (int) Math.ceil(15.0 / compactionThreshold);
  byte[] pad = new byte[1000]; // 1 KB chunk
  for (int i = 0; i < compactionThreshold; i++) {
    Table loader = new RegionAsTable(r);
    Put p = new Put(Bytes.add(STARTROW, Bytes.toBytes(i)));
    p.setDurability(Durability.SKIP_WAL);
    for (int j = 0; j < jmax; j++) {
      p.addColumn(COLUMN_FAMILY, Bytes.toBytes(j), pad);
    }
    HTestConst.addContent(loader, Bytes.toString(COLUMN_FAMILY));
    loader.put(p);
    r.flush(true);
  }
  HStore s = r.getStore(COLUMN_FAMILY);
  int initialFiles = s.getStorefilesCount();

  thread.requestCompaction(r, store, "test custom comapction", PRIORITY_USER,
      CompactionLifeCycleTracker.DUMMY, null);

  Thread.sleep(3000);
  thread.switchCompaction(false);
  assertEquals(initialFiles, s.getStorefilesCount());
  //don't mess up future tests
  thread.switchCompaction(true);
}
 
Example 26
Source Project: hbase   Source File: TestHRegionServerBulkLoad.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request)
    throws IOException {
  try {
    Thread.sleep(sleepDuration);
  } catch (InterruptedException ie) {
    IOException ioe = new InterruptedIOException();
    ioe.initCause(ie);
    throw ioe;
  }
  return scanner;
}
 
Example 27
Source Project: hbase   Source File: TestCompactionLifeCycleTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> candidates, CompactionLifeCycleTracker tracker)
    throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}
 
Example 28
Source Project: hbase   Source File: TestCompactionLifeCycleTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}
 
Example 29
Source Project: hbase   Source File: TestCompactionLifeCycleTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
  return scanner;
}
 
Example 30
Source Project: hbase   Source File: TestCompactionLifeCycleTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
    throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}