org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils Java Examples

The following examples show how to use org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RestoreTool.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Returns value represent path for:
 * ""/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot/
 *    snapshot_1396650097621_namespace_table"
 * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo,
 * .data.manifest (trunk)
 * @param tableName table name
 * @return path to table info
 * @throws IOException exception
 */
Path getTableInfoPath(TableName tableName) throws IOException {
  Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
  Path tableInfoPath = null;

  // can't build the path directly as the timestamp values are different
  FileStatus[] snapshots = fs.listStatus(tableSnapShotPath,
      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
  for (FileStatus snapshot : snapshots) {
    tableInfoPath = snapshot.getPath();
    // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest";
    if (tableInfoPath.getName().endsWith("data.manifest")) {
      break;
    }
  }
  return tableInfoPath;
}
 
Example #2
Source File: MapReduceParallelScanGrouper.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
	String snapshotName;
	Configuration conf = context.getConnection().getQueryServices().getConfiguration();
	if((snapshotName = getSnapshotName(conf)) != null) {
		try {
			Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
			FileSystem fs = rootDir.getFileSystem(conf);
			Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
			SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
			SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
			return getRegionLocationsFromManifest(manifest);
		}
		catch (Exception e) {
			throw new RuntimeException(e);
		}
	}
	else {
		return context.getConnection().getQueryServices().getAllTableRegions(tableName);
	}
}
 
Example #3
Source File: TestSnapshotHFileCleaner.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException,
 * instead of an IOException
 */
@Test
public void testCorruptedDataManifest() throws IOException {
  SnapshotTestingUtils.SnapshotMock
      snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
  SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
  builder.addRegionV2();
  // consolidate to generate a data.manifest file
  builder.consolidate();
  builder.corruptDataManifest();

  long period = Long.MAX_VALUE;
  SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());
  try {
    cache.getSnapshotsInProgress();
  } finally {
    fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir,
        TEST_UTIL.getConfiguration()), true);
  }
}
 
Example #4
Source File: TestSnapshotHFileCleaner.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
 * instead of an IOException
 */
@Test
public void testCorruptedRegionManifest() throws IOException {
  SnapshotTestingUtils.SnapshotMock
      snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
  SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
  builder.addRegionV2();
  builder.corruptOneRegionManifest();

  long period = Long.MAX_VALUE;
  SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());
  try {
    cache.getSnapshotsInProgress();
  } finally {
    fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf), true);
  }
}
 
Example #5
Source File: TestFileArchiverNotifierImpl.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 
Example #6
Source File: TestSnapshotDFSTemporaryDirectory.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) throws IOException {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());

  String snapshotPath = UTIL.getDefaultRootDirPath().toString() + Path.SEPARATOR +
      UUID.randomUUID().toString() + Path.SEPARATOR + ".tmpdir" + Path.SEPARATOR;
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(snapshotPath).toUri());
}
 
Example #7
Source File: TestSnapshotTemporaryDirectory.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(TEMP_DIR, ".tmpDir").toUri());
}
 
Example #8
Source File: SnapshotHFileCleaner.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void setConf(final Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
      DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
    final FileSystem fs = CommonFSUtils.getCurrentFileSystem(conf);
    Path rootDir = CommonFSUtils.getRootDir(conf);
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf);
    FileSystem workingFs = workingDir.getFileSystem(conf);

    cache = new SnapshotFileCache(fs, rootDir, workingFs, workingDir, cacheRefreshPeriod,
      cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher",
      new SnapshotFileCache.SnapshotFileInspector() {
          @Override
          public Collection<String> filesUnderSnapshot(final FileSystem fs,
            final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create cleaner util", e);
  }
}
 
Example #9
Source File: SnapshotManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Take a snapshot using the specified handler.
 * On failure the snapshot temporary working directory is removed.
 * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
 *       snapshot request if the table is busy with another snapshot/restore operation.
 * @param snapshot the snapshot description
 * @param handler the snapshot handler
 */
private synchronized void snapshotTable(SnapshotDescription snapshot,
    final TakeSnapshotHandler handler) throws IOException {
  try {
    handler.prepare();
    this.executorService.submit(handler);
    this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
  } catch (Exception e) {
    // cleanup the working directory by trying to delete it from the fs.
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir,
        master.getConfiguration());
    FileSystem workingDirFs = workingDir.getFileSystem(master.getConfiguration());
    try {
      if (!workingDirFs.delete(workingDir, true)) {
        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
            ClientSnapshotDescriptionUtils.toString(snapshot));
      }
    } catch (IOException e1) {
      LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
          ClientSnapshotDescriptionUtils.toString(snapshot));
    }
    // fail the snapshot
    throw new SnapshotCreationException("Could not build snapshot handler", e,
      ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 
Example #10
Source File: CloneSnapshotProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Action before cloning from snapshot.
 * @param env MasterProcedureEnv
 * @throws IOException
 * @throws InterruptedException
 */
private void preCloneSnapshot(final MasterProcedureEnv env)
    throws IOException, InterruptedException {
  if (!getTableName().isSystemTable()) {
    // Check and update namespace quota
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();

    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterConfiguration(),
      mfs.getFileSystem(),
      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
      snapshot);

    ProcedureSyncWait.getMasterQuotaManager(env)
      .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
  }

  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preCreateTableAction(tableDescriptor, null, getUser());
  }
}
 
Example #11
Source File: Utils.java    From presto-hbase-connector with Apache License 2.0 6 votes vote down vote up
/**
 * get region infos
 *
 * @param zookeeperQuorum     zookeeper quorum
 * @param zookeeperClientPort zookeeper client port
 * @param hBaseRootDir        HBase root dir
 * @param snapshotName        snapshot name
 * @return region info list
 * @throws IOException IOException
 */
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
                                               String hBaseRootDir, String snapshotName) throws IOException {
    try {
        Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
        Path root = new Path(hBaseRootDir);
        FileSystem fs = FileSystem.get(conf);
        Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
        HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
        return Utils.getRegionInfosFromManifest(manifest);
    } catch (IOException ex) {
        logger.error("get region info error: " + ex.getMessage(), ex);
        throw ex;
    }
}
 
Example #12
Source File: RestoreTool.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Get table descriptor
 * @param tableName is the table backed up
 * @return {@link TableDescriptor} saved in backup image of the table
 */
TableDescriptor getTableDesc(TableName tableName) throws IOException {
  Path tableInfoPath = this.getTableInfoPath(tableName);
  SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
  TableDescriptor tableDescriptor = manifest.getTableDescriptor();
  if (!tableDescriptor.getTableName().equals(tableName)) {
    LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
            + tableInfoPath.toString());
    LOG.error("tableDescriptor.getNameAsString() = "
            + tableDescriptor.getTableName().getNameAsString());
    throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
        + " under tableInfoPath: " + tableInfoPath.toString());
  }
  return tableDescriptor;
}
 
Example #13
Source File: AccessController.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)
      && hTableDescriptor.getTableName().getNameAsString()
      .equals(snapshot.getTableNameAsString())) {
    // Snapshot owner is allowed to create a table with the same name as the snapshot he took
    AuthResult result = AuthResult.allow("cloneSnapshot " + snapshot.getName(),
      "Snapshot owner check allowed", user, null, hTableDescriptor.getTableName(), null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "cloneSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example #14
Source File: TableSnapshotScanner.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void openWithoutRestoringSnapshot() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotProtos.SnapshotDescription snapshotDesc =
      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
  }

  regions = new ArrayList<>(regionManifests.size());
  regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
    .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
  htd = manifest.getTableDescriptor();
}
 
Example #15
Source File: RestoreSnapshotProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Execute the on-disk Restore
 * @param env MasterProcedureEnv
 * @throws IOException
 **/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
  MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
  FileSystem fs = fileSystemManager.getFileSystem();
  Path rootDir = fileSystemManager.getRootDir();
  final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();

  LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
  try {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
    RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
      env.getMasterServices().getConfiguration(),
      fs,
      manifest,
      modifiedTableDescriptor,
      rootDir,
      monitorException,
      getMonitorStatus());

    RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
    regionsToRestore = metaChanges.getRegionsToRestore();
    regionsToRemove = metaChanges.getRegionsToRemove();
    regionsToAdd = metaChanges.getRegionsToAdd();
    parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap();
  } catch (IOException e) {
    String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
      + " failed in on-disk restore. Try re-running the restore command.";
    LOG.error(msg, e);
    monitorException.receive(
      new ForeignException(env.getMasterServices().getServerName().toString(), e));
    throw new IOException(msg, e);
  }
}
 
Example #16
Source File: TestSnapshotFileCacheWithDifferentWorkingDir.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void startCluster() throws Exception {
  initCommon();

  // Set the snapshot working directory to be on another filesystem.
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR,
    "file://" + new Path(TEMP_DIR, ".tmpDir").toUri());
  workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf);
  workingFs = workingDir.getFileSystem(conf);
}
 
Example #17
Source File: TestSnapshotHFileCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testFindsSnapshotFilesWhenCleaning() throws IOException {
  CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);

  FileSystem fs = FileSystem.get(conf);
  SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
  cleaner.setConf(conf);

  // write an hfile to the snapshot directory
  String snapshotName = "snapshot";
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  RegionInfo mockRegion = RegionInfoBuilder.newBuilder(tableName).build();
  Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
  Path familyDir = new Path(regionSnapshotDir, "family");
  // create a reference to a supposedly valid hfile
  String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
  Path refFile = new Path(familyDir, hfile);

  // make sure the reference file exists
  fs.create(refFile);

  // create the hfile in the archive
  fs.mkdirs(archivedHfileDir);
  fs.createNewFile(new Path(archivedHfileDir, hfile));

  // make sure that the file isn't deletable
  assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
}
 
Example #18
Source File: TestSnapshotFileCache.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected static void initCommon() throws Exception {
  UTIL.startMiniDFSCluster(1);
  fs = UTIL.getDFSCluster().getFileSystem();
  rootDir = UTIL.getDefaultRootDirPath();
  snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  conf = UTIL.getConfiguration();
}
 
Example #19
Source File: AccessController.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // list it, if user is the owner of snapshot
    AuthResult result = AuthResult.allow("listSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "listSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example #20
Source File: AccessController.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    accessChecker.requirePermission(user, "restoreSnapshot " + snapshot.getName(),
      hTableDescriptor.getTableName(), null, null, null, Permission.Action.ADMIN);
  } else {
    accessChecker.requirePermission(user, "restoreSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example #21
Source File: AccessController.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // Snapshot owner is allowed to delete the snapshot
    AuthResult result = AuthResult.allow("deleteSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "deleteSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example #22
Source File: SnapshotManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
    IOException, UnsupportedOperationException {
  this.master = master;

  this.rootDir = master.getMasterFileSystem().getRootDir();
  checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());

  // get the configuration for the coordinator
  Configuration conf = master.getConfiguration();
  long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
  long timeoutMillis = Math.max(
          conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
                  SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME),
          conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
                  SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME));
  int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);

  // setup the default procedure coordinator
  String name = master.getServerName().toString();
  ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
  ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinator(
      master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);

  this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
  this.executorService = master.getExecutorService();
  resetTempDir();
  snapshotHandlerChoreCleanerTask =
      scheduleThreadPool.scheduleAtFixedRate(this::cleanupSentinels, 10, 10, TimeUnit.SECONDS);
}
 
Example #23
Source File: SnapshotManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Check to see if the snapshot is one of the currently completed snapshots
 * Returns true if the snapshot exists in the "completed snapshots folder".
 *
 * @param snapshot expected snapshot to check
 * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
 *         not stored
 * @throws IOException if the filesystem throws an unexpected exception,
 * @throws IllegalArgumentException if snapshot name is invalid.
 */
private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
  try {
    final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    // check to see if the snapshot already exists
    return fs.exists(snapshotDir);
  } catch (IllegalArgumentException iae) {
    throw new UnknownSnapshotException("Unexpected exception thrown", iae);
  }
}
 
Example #24
Source File: SnapshotManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Delete the specified snapshot
 * @param snapshot
 * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
 * @throws IOException For filesystem IOExceptions
 */
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
  // check to see if it is completed
  if (!isSnapshotCompleted(snapshot)) {
    throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
  }

  String snapshotName = snapshot.getName();
  // first create the snapshot description and check to see if it exists
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
  // just the "name" and it does not contains the "real" snapshot information
  snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  // call coproc pre hook
  MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
  org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
  if (cpHost != null) {
    snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
    cpHost.preDeleteSnapshot(snapshotPOJO);
  }

  LOG.debug("Deleting snapshot: " + snapshotName);
  // delete the existing snapshot
  if (!fs.delete(snapshotDir, true)) {
    throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
  }

  // call coproc post hook
  if (cpHost != null) {
    cpHost.postDeleteSnapshot(snapshotPOJO);
  }

}
 
Example #25
Source File: SnapshotManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
 * snapshot attempts.
 *
 * @throws IOException if we can't reach the filesystem
 */
private void resetTempDir() throws IOException {
  // cleanup any existing snapshots.
  Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir,
      master.getConfiguration());
  FileSystem tmpFs = tmpdir.getFileSystem(master.getConfiguration());
  if (!tmpFs.delete(tmpdir, true)) {
    LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
  }
}
 
Example #26
Source File: MasterSnapshotVerifier.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Check that the snapshot description written in the filesystem matches the current snapshot
 * @param snapshotDir snapshot directory to check
 */
private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
  SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(workingDirFs,
      snapshotDir);
  if (!this.snapshot.equals(found)) {
    throw new CorruptedSnapshotException(
        "Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").",
        ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 
Example #27
Source File: SnapshotFileCache.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
 * filesystem
 * @param fs {@link FileSystem} where the snapshots are stored
 * @param rootDir hbase root directory
 * @param workingFs {@link FileSystem} where ongoing snapshot mainifest files are stored
 * @param workingDir Location to store ongoing snapshot manifest files
 * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
 * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
 * @param refreshThreadName name of the cache refresh thread
 * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
 */
public SnapshotFileCache(FileSystem fs, Path rootDir, FileSystem workingFs, Path workingDir,
  long cacheRefreshPeriod, long cacheRefreshDelay, String refreshThreadName,
  SnapshotFileInspector inspectSnapshotFiles) {
  this.fs = fs;
  this.workingFs = workingFs;
  this.workingSnapshotDir = workingDir;
  this.fileInspector = inspectSnapshotFiles;
  this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  // periodically refresh the file cache to make sure we aren't superfluously saving files.
  this.refreshTimer = new Timer(refreshThreadName, true);
  this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
    cacheRefreshPeriod);
}
 
Example #28
Source File: FileArchiverNotifierImpl.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * For the given snapshot, find all files which this {@code snapshotName} references. After a file
 * is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
 * {@code snapshotSizeChanges} is updated in concert.
 *
 * @param snapshotName The snapshot to check
 * @param filesToUpdate A mapping of archived files to their size
 * @param snapshotSizeChanges A mapping of snapshots and their change in size
 */
void bucketFilesToSnapshot(
    String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
        throws IOException {
  // A quick check to avoid doing work if the caller unnecessarily invoked this method.
  if (filesToUpdate.isEmpty()) {
    return;
  }

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        Long valueOrNull = filesToUpdate.remove(sf.getName());
        if (valueOrNull != null) {
          // This storefile was recently archived, we should update this snapshot with its size
          snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
        }
        // Short-circuit, if we have no more files that were archived, we don't need to iterate
        // over the rest of the snapshot.
        if (filesToUpdate.isEmpty()) {
          return;
        }
      }
    }
  }
}
 
Example #29
Source File: CloneSnapshotProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException {
  Configuration conf = env.getMasterServices().getConfiguration();
  if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
      && SnapshotDescriptionUtils.isSecurityAvailable(conf)) {
    RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, tableDescriptor.getTableName(), conf);
  }
}
 
Example #30
Source File: RestoreSnapshotProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void restoreSnapshotAcl(final MasterProcedureEnv env) throws IOException {
  if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
      && SnapshotDescriptionUtils
          .isSecurityAvailable(env.getMasterServices().getConfiguration())) {
    // restore acl of snapshot to table.
    RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, TableName.valueOf(snapshot.getTable()),
      env.getMasterServices().getConfiguration());
  }
}