org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest Java Examples

The following examples show how to use org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #2
Source File: ServerSideOperationsObserver.java    From geowave with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompact(
    final ObserverContext<RegionCoprocessorEnvironment> e,
    final Store store,
    final InternalScanner scanner,
    final ScanType scanType,
    final CompactionRequest request) throws IOException {
  if (opStore == null) {
    return super.preCompact(e, store, scanner, scanType, request);
  }
  return super.preCompact(
      e,
      store,
      wrapScannerWithOps(
          e.getEnvironment().getRegionInfo().getTable(),
          scanner,
          null,
          ServerOpScope.MAJOR_COMPACTION,
          INTERNAL_SCANNER_FACTORY),
      scanType,
      request);
}
 
Example #3
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #4
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #5
Source File: MemstoreAwareObserver.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
                                  InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
                                  CompactionRequest request) throws IOException {
    try {
        BlockingProbe.blockPreCompact();
        if (!(request instanceof SpliceCompactionRequest)) {
            SpliceLogUtils.error(LOG,"Compaction request must be a SpliceCompactionRequest");
            throw new DoNotRetryIOException();
        }
        SpliceCompactionRequest scr = (SpliceCompactionRequest) request;
        // memstoreAware is injected into the request, where the blocking logic lives, and where compaction
        // count will be incremented and decremented.
        scr.setMemstoreAware(memstoreAware);
        HRegion region = (HRegion) c.getEnvironment().getRegion();
        scr.setRegion(region);
        return scanner == null ? DummyScanner.INSTANCE : scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example #6
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompact(
    org.apache.hadoop.hbase.coprocessor.ObserverContext<RegionCoprocessorEnvironment> c,
    Store store, InternalScanner scanner, ScanType scanType,
    org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();
  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }
  // Also make sure to use the same snapshot for the compaction
  InternalScanner s =
      createStoreScanner(c.getEnvironment(), "compaction", snapshot, scanner, scanType);
  if (s != null) {
    return s;
  }
  return scanner;
}
 
Example #7
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #8
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #9
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #10
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile,
						CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postCompact()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postCompact(c, store, resultFile, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postCompact()");
	}
}
 
Example #11
Source File: TransactionProcessor.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends KeyValueScanner> scanners, ScanType scanType, long earliestPutTs, InternalScanner s,
    CompactionRequest request)
    throws IOException {
  // Get the latest tx snapshot state for the compaction
  TransactionVisibilityState snapshot = cache.getLatestState();

  // Record tx state before the compaction
  if (compactionState != null) {
    compactionState.record(request, snapshot);
  }

  // silently close the passed scanner as we're returning a brand-new one
  try (InternalScanner temp = s) { }
  // Also make sure to use the same snapshot for the compaction
  return createStoreScanner(c.getEnvironment(), "compaction", snapshot, store, scanners, scanType, earliestPutTs);
}
 
Example #12
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,	InternalScanner scanner,
								  ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {

	final InternalScanner ret;

	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.preCompact()");
	}

	try {
		activatePluginClassLoader();
		ret = implRegionObserver.preCompact(c, store, scanner, scanType, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.preCompact()");
	}

	return ret;
}
 
Example #13
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store, List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker, CompactionRequest request) {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postCompactSelection()");
	}

	try {
		activatePluginClassLoader();
		implRegionObserver.postCompactSelection(c, store, selected, tracker, request);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postCompactSelection()");
	}
}
 
Example #14
Source File: IndexHalfStoreFileReaderGenerator.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public InternalScanner preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c,
        Store store, List<? extends KeyValueScanner> scanners, ScanType scanType,
        long earliestPutTs, InternalScanner s, CompactionRequest request) throws IOException {
    InternalScanner internalScanner = super.preCompactScannerOpen(c, store, scanners, scanType, earliestPutTs, s, request);
    Collection<StoreFile> files = request.getFiles();
    storeFilesCount = 0;
    compactedFilesCount = 0;
    for(StoreFile file:files) {
        if(!file.isReference()) {
            return internalScanner;
        }
    }
    storeFilesCount = files.size();
    return internalScanner;
}
 
Example #15
Source File: TestCoprocessorInterface.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e,
    Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  preCompactCalled = true;
  return scanner;
}
 
Example #16
Source File: TestHRegionServerBulkLoad.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> e, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request)
    throws IOException {
  try {
    Thread.sleep(sleepDuration);
  } catch (InterruptedException ie) {
    IOException ioe = new InterruptedIOException();
    ioe.initCause(ie);
    throw ioe;
  }
  return scanner;
}
 
Example #17
Source File: TestCompactionLifeCycleTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompactSelection(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}
 
Example #18
Source File: TestCompactionLifeCycleTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
  return scanner;
}
 
Example #19
Source File: TestCompactionLifeCycleTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request)
    throws IOException {
  if (TRACKER != null) {
    assertSame(tracker, TRACKER);
  }
}
 
Example #20
Source File: IndexHalfStoreFileReaderGenerator.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
        InternalScanner s, ScanType scanType, CompactionLifeCycleTracker tracker,
        CompactionRequest request) throws IOException {

    if (!IndexUtil.isLocalIndexStore(store)) { return s; }
    if (!store.hasReferences()) {
        InternalScanner repairScanner = null;
        if (request.isMajor() && (!RepairUtil.isLocalIndexStoreFilesConsistent(c.getEnvironment(), store))) {
            LOGGER.info("we have found inconsistent data for local index for region:"
                    + c.getEnvironment().getRegion().getRegionInfo());
            if (c.getEnvironment().getConfiguration().getBoolean(LOCAL_INDEX_AUTOMATIC_REPAIR, true)) {
                LOGGER.info("Starting automatic repair of local Index for region:"
                        + c.getEnvironment().getRegion().getRegionInfo());
                repairScanner = getRepairScanner(c.getEnvironment(), store);
            }
        }
        if (repairScanner != null) {
            if (s!=null) {
                s.close();
            }
            return repairScanner;
        } else {
            return s;
        }
    }
    return s;
}
 
Example #21
Source File: ExampleRegionObserverWithMetrics.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void postCompactSelection(
    ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    List<? extends StoreFile> selected, CompactionLifeCycleTracker tracker,
    CompactionRequest request) {
  if (selected != null) {
    filesCompactedCounter.increment(selected.size());
  }
}
 
Example #22
Source File: AccessController.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store,
    InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker,
    CompactionRequest request) throws IOException {
  requirePermission(c, "compact", getTableName(c.getEnvironment()),
      null, null, Action.ADMIN, Action.CREATE);
  return scanner;
}
 
Example #23
Source File: RegionCoprocessorHost.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Called prior to opening store scanner for compaction.
 */
public ScanInfo preCompactScannerOpen(HStore store, ScanType scanType,
    CompactionLifeCycleTracker tracker, CompactionRequest request, User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return store.getScanInfo();
  }
  CustomizedScanInfoBuilder builder = new CustomizedScanInfoBuilder(store.getScanInfo());
  execOperation(new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.preCompactScannerOpen(this, store, scanType, builder, tracker, request);
    }
  });
  return builder.build();
}
 
Example #24
Source File: RegionCoprocessorHost.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Called after the {@link HStoreFile}s to be compacted have been selected from the available
 * candidates.
 * @param store The store where compaction is being requested
 * @param selected The store files selected to compact
 * @param tracker used to track the life cycle of a compaction
 * @param request the compaction request
 * @param user the user
 */
public void postCompactSelection(final HStore store, final List<HStoreFile> selected,
    final CompactionLifeCycleTracker tracker, final CompactionRequest request,
    final User user) throws IOException {
  if (coprocEnvironments.isEmpty()) {
    return;
  }
  execOperation(new RegionObserverOperationWithoutResult(user) {
    @Override
    public void call(RegionObserver observer) throws IOException {
      observer.postCompactSelection(this, store, selected, tracker, request);
    }
  });
}
 
Example #25
Source File: BaseRegionObserver.java    From phoenix-omid with Apache License 2.0 5 votes vote down vote up
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c,
                                  Store store,
                                  InternalScanner scanner,
                                  ScanType scanType,
                                  CompactionLifeCycleTracker tracker,
                                  CompactionRequest request) throws IOException {
    return preCompact(c,store,scanner,scanType,request);
}
 
Example #26
Source File: SIObserver.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        if(tableEnvMatch){
            Tracer.compact();
        }
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example #27
Source File: SIObserver.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        // We can't return null, there's a check in org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost.preCompact
        // return a dummy implementation instead
        if (scanner == null || scanner == DummyScanner.INSTANCE)
            return DummyScanner.INSTANCE;

        if(tableEnvMatch){
            SIDriver driver=SIDriver.driver();
            SimpleCompactionContext context = new SimpleCompactionContext();
            SICompactionState state = new SICompactionState(driver.getTxnSupplier(),
                    driver.getConfiguration().getActiveTransactionMaxCacheSize(), context, driver.getRejectingExecutorService());
            SConfiguration conf = driver.getConfiguration();
            PurgeConfig purgeConfig;
            if (conf.getOlapCompactionAutomaticallyPurgeDeletedRows()) {
                if (request.isMajor())
                    purgeConfig = PurgeConfig.purgeDuringMajorCompactionConfig();
                else
                    purgeConfig = PurgeConfig.purgeDuringMinorCompactionConfig();
            } else {
                purgeConfig = PurgeConfig.noPurgeConfig();
            }
            SICompactionScanner siScanner = new SICompactionScanner(
                    state, scanner, purgeConfig,
                    conf.getOlapCompactionResolutionShare(), conf.getLocalCompactionResolutionBufferSize(), context);
            siScanner.start();
            return siScanner;
        }
        return scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example #28
Source File: BackupEndpointObserver.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public InternalScanner preCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, InternalScanner scanner, ScanType scanType, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        if (LOG.isDebugEnabled())
            SpliceLogUtils.debug(LOG, "BackupEndpointObserver.preCompact()");

        BackupUtils.waitForBackupToComplete(tableName, regionName);
        isCompacting.set(true);
        SpliceLogUtils.info(LOG, "setting isCompacting=true for %s:%s", tableName, regionName);
        return scanner == null ? DummyScanner.INSTANCE : scanner;
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example #29
Source File: BackupEndpointObserver.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void postCompact(ObserverContext<RegionCoprocessorEnvironment> c, Store store, StoreFile resultFile, CompactionLifeCycleTracker tracker, CompactionRequest request) throws IOException {
    try {
        isCompacting.set(false);
        SpliceLogUtils.info(LOG, "setting isCompacting=false for %s:%s", tableName, regionName);
        if (LOG.isDebugEnabled()) {
            String filePath =  resultFile != null?resultFile.getPath().toString():null;
            SpliceLogUtils.debug(LOG, "Compaction result file %s", filePath);
        }
    } catch (Throwable t) {
        throw CoprocessorUtils.getIOException(t);
    }
}
 
Example #30
Source File: TestRegionCoprocessorHost.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreCompactScannerOpen() throws IOException {
  RegionCoprocessorHost host = new RegionCoprocessorHost(region, rsServices, conf);
  ScanInfo oldScanInfo = getScanInfo();
  HStore store = mock(HStore.class);
  when(store.getScanInfo()).thenReturn(oldScanInfo);
  ScanInfo newScanInfo = host.preCompactScannerOpen(store, ScanType.COMPACT_DROP_DELETES,
    mock(CompactionLifeCycleTracker.class), mock(CompactionRequest.class), mock(User.class));
  verifyScanInfo(newScanInfo);
}