Java Code Examples for org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils

The following examples show how to use org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto-hbase-connector   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * get region infos
 *
 * @param zookeeperQuorum     zookeeper quorum
 * @param zookeeperClientPort zookeeper client port
 * @param hBaseRootDir        HBase root dir
 * @param snapshotName        snapshot name
 * @return region info list
 * @throws IOException IOException
 */
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
                                               String hBaseRootDir, String snapshotName) throws IOException {
    try {
        Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
        Path root = new Path(hBaseRootDir);
        FileSystem fs = FileSystem.get(conf);
        Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
        HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
        return Utils.getRegionInfosFromManifest(manifest);
    } catch (IOException ex) {
        logger.error("get region info error: " + ex.getMessage(), ex);
        throw ex;
    }
}
 
Example 2
Source Project: hbase   Source File: RestoreTool.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns value represent path for:
 * ""/$USER/SBACKUP_ROOT/backup_id/namespace/table/.hbase-snapshot/
 *    snapshot_1396650097621_namespace_table"
 * this path contains .snapshotinfo, .tabledesc (0.96 and 0.98) this path contains .snapshotinfo,
 * .data.manifest (trunk)
 * @param tableName table name
 * @return path to table info
 * @throws IOException exception
 */
Path getTableInfoPath(TableName tableName) throws IOException {
  Path tableSnapShotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
  Path tableInfoPath = null;

  // can't build the path directly as the timestamp values are different
  FileStatus[] snapshots = fs.listStatus(tableSnapShotPath,
      new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs));
  for (FileStatus snapshot : snapshots) {
    tableInfoPath = snapshot.getPath();
    // SnapshotManifest.DATA_MANIFEST_NAME = "data.manifest";
    if (tableInfoPath.getName().endsWith("data.manifest")) {
      break;
    }
  }
  return tableInfoPath;
}
 
Example 3
Source Project: hbase   Source File: RestoreTool.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get table descriptor
 * @param tableName is the table backed up
 * @return {@link TableDescriptor} saved in backup image of the table
 */
TableDescriptor getTableDesc(TableName tableName) throws IOException {
  Path tableInfoPath = this.getTableInfoPath(tableName);
  SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
  TableDescriptor tableDescriptor = manifest.getTableDescriptor();
  if (!tableDescriptor.getTableName().equals(tableName)) {
    LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
            + tableInfoPath.toString());
    LOG.error("tableDescriptor.getNameAsString() = "
            + tableDescriptor.getTableName().getNameAsString());
    throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
        + " under tableInfoPath: " + tableInfoPath.toString());
  }
  return tableDescriptor;
}
 
Example 4
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void preCloneSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)
      && hTableDescriptor.getTableName().getNameAsString()
      .equals(snapshot.getTableNameAsString())) {
    // Snapshot owner is allowed to create a table with the same name as the snapshot he took
    AuthResult result = AuthResult.allow("cloneSnapshot " + snapshot.getName(),
      "Snapshot owner check allowed", user, null, hTableDescriptor.getTableName(), null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "cloneSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 5
Source Project: hbase   Source File: TableSnapshotScanner.java    License: Apache License 2.0 6 votes vote down vote up
private void openWithoutRestoringSnapshot() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotProtos.SnapshotDescription snapshotDesc =
      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
  }

  regions = new ArrayList<>(regionManifests.size());
  regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
    .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
  htd = manifest.getTableDescriptor();
}
 
Example 6
Source Project: hbase   Source File: CloneSnapshotProcedure.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Action before cloning from snapshot.
 * @param env MasterProcedureEnv
 * @throws IOException
 * @throws InterruptedException
 */
private void preCloneSnapshot(final MasterProcedureEnv env)
    throws IOException, InterruptedException {
  if (!getTableName().isSystemTable()) {
    // Check and update namespace quota
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();

    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterConfiguration(),
      mfs.getFileSystem(),
      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
      snapshot);

    ProcedureSyncWait.getMasterQuotaManager(env)
      .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
  }

  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preCreateTableAction(tableDescriptor, null, getUser());
  }
}
 
Example 7
Source Project: hbase   Source File: SnapshotManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Take a snapshot using the specified handler.
 * On failure the snapshot temporary working directory is removed.
 * NOTE: prepareToTakeSnapshot() called before this one takes care of the rejecting the
 *       snapshot request if the table is busy with another snapshot/restore operation.
 * @param snapshot the snapshot description
 * @param handler the snapshot handler
 */
private synchronized void snapshotTable(SnapshotDescription snapshot,
    final TakeSnapshotHandler handler) throws IOException {
  try {
    handler.prepare();
    this.executorService.submit(handler);
    this.snapshotHandlers.put(TableName.valueOf(snapshot.getTable()), handler);
  } catch (Exception e) {
    // cleanup the working directory by trying to delete it from the fs.
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(snapshot, rootDir,
        master.getConfiguration());
    FileSystem workingDirFs = workingDir.getFileSystem(master.getConfiguration());
    try {
      if (!workingDirFs.delete(workingDir, true)) {
        LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
            ClientSnapshotDescriptionUtils.toString(snapshot));
      }
    } catch (IOException e1) {
      LOG.error("Couldn't delete working directory (" + workingDir + " for snapshot:" +
          ClientSnapshotDescriptionUtils.toString(snapshot));
    }
    // fail the snapshot
    throw new SnapshotCreationException("Could not build snapshot handler", e,
      ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 
Example 8
Source Project: hbase   Source File: SnapshotHFileCleaner.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setConf(final Configuration conf) {
  super.setConf(conf);
  try {
    long cacheRefreshPeriod = conf.getLong(HFILE_CACHE_REFRESH_PERIOD_CONF_KEY,
      DEFAULT_HFILE_CACHE_REFRESH_PERIOD);
    final FileSystem fs = CommonFSUtils.getCurrentFileSystem(conf);
    Path rootDir = CommonFSUtils.getRootDir(conf);
    Path workingDir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf);
    FileSystem workingFs = workingDir.getFileSystem(conf);

    cache = new SnapshotFileCache(fs, rootDir, workingFs, workingDir, cacheRefreshPeriod,
      cacheRefreshPeriod, "snapshot-hfile-cleaner-cache-refresher",
      new SnapshotFileCache.SnapshotFileInspector() {
          @Override
          public Collection<String> filesUnderSnapshot(final FileSystem fs,
            final Path snapshotDir)
              throws IOException {
            return SnapshotReferenceUtil.getHFileNames(conf, fs, snapshotDir);
          }
        });
  } catch (IOException e) {
    LOG.error("Failed to create cleaner util", e);
  }
}
 
Example 9
Source Project: hbase   Source File: TestSnapshotTemporaryDirectory.java    License: Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(TEMP_DIR, ".tmpDir").toUri());
}
 
Example 10
Source Project: hbase   Source File: TestSnapshotDFSTemporaryDirectory.java    License: Apache License 2.0 6 votes vote down vote up
private static void setupConf(Configuration conf) throws IOException {
  // disable the ui
  conf.setInt("hbase.regionsever.info.port", -1);
  // change the flush size to a small amount, regulating number of store files
  conf.setInt("hbase.hregion.memstore.flush.size", 25000);
  // so make sure we get a compaction when doing a load, but keep around some
  // files in the store
  conf.setInt("hbase.hstore.compaction.min", 10);
  conf.setInt("hbase.hstore.compactionThreshold", 10);
  // block writes if we get to 12 store files
  conf.setInt("hbase.hstore.blockingStoreFiles", 12);
  // Enable snapshot
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.set(HConstants.HBASE_REGION_SPLIT_POLICY_KEY,
      ConstantSizeRegionSplitPolicy.class.getName());

  String snapshotPath = UTIL.getDefaultRootDirPath().toString() + Path.SEPARATOR +
      UUID.randomUUID().toString() + Path.SEPARATOR + ".tmpdir" + Path.SEPARATOR;
  conf.set(SnapshotDescriptionUtils.SNAPSHOT_WORKING_DIR, "file://" + new Path(snapshotPath).toUri());
}
 
Example 11
Source Project: hbase   Source File: TestFileArchiverNotifierImpl.java    License: Apache License 2.0 6 votes vote down vote up
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 
Example 12
Source Project: hbase   Source File: TestSnapshotHFileCleaner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * If there is a corrupted region manifest, it should throw out CorruptedSnapshotException,
 * instead of an IOException
 */
@Test
public void testCorruptedRegionManifest() throws IOException {
  SnapshotTestingUtils.SnapshotMock
      snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
  SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
  builder.addRegionV2();
  builder.corruptOneRegionManifest();

  long period = Long.MAX_VALUE;
  SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());
  try {
    cache.getSnapshotsInProgress();
  } finally {
    fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir, conf), true);
  }
}
 
Example 13
Source Project: hbase   Source File: TestSnapshotHFileCleaner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * If there is a corrupted data manifest, it should throw out CorruptedSnapshotException,
 * instead of an IOException
 */
@Test
public void testCorruptedDataManifest() throws IOException {
  SnapshotTestingUtils.SnapshotMock
      snapshotMock = new SnapshotTestingUtils.SnapshotMock(conf, fs, rootDir);
  SnapshotTestingUtils.SnapshotMock.SnapshotBuilder builder = snapshotMock.createSnapshotV2(
      SNAPSHOT_NAME_STR, TABLE_NAME_STR);
  builder.addRegionV2();
  // consolidate to generate a data.manifest file
  builder.consolidate();
  builder.corruptDataManifest();

  long period = Long.MAX_VALUE;
  SnapshotFileCache cache = new SnapshotFileCache(conf, period, 10000000,
      "test-snapshot-file-cache-refresh", new SnapshotFiles());
  try {
    cache.getSnapshotsInProgress();
  } finally {
    fs.delete(SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir,
        TEST_UTIL.getConfiguration()), true);
  }
}
 
Example 14
Source Project: phoenix   Source File: MapReduceParallelScanGrouper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
	String snapshotName;
	Configuration conf = context.getConnection().getQueryServices().getConfiguration();
	if((snapshotName = getSnapshotName(conf)) != null) {
		try {
			Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
			FileSystem fs = rootDir.getFileSystem(conf);
			Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
			SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
			SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
			return getRegionLocationsFromManifest(manifest);
		}
		catch (Exception e) {
			throw new RuntimeException(e);
		}
	}
	else {
		return context.getConnection().getQueryServices().getAllTableRegions(tableName);
	}
}
 
Example 15
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preListSnapshot(ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // list it, if user is the owner of snapshot
    AuthResult result = AuthResult.allow("listSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "listSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 16
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preRestoreSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot, final TableDescriptor hTableDescriptor)
    throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    accessChecker.requirePermission(user, "restoreSnapshot " + snapshot.getName(),
      hTableDescriptor.getTableName(), null, null, null, Permission.Action.ADMIN);
  } else {
    accessChecker.requirePermission(user, "restoreSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 17
Source Project: hbase   Source File: AccessController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preDeleteSnapshot(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    final SnapshotDescription snapshot) throws IOException {
  User user = getActiveUser(ctx);
  if (SnapshotDescriptionUtils.isSnapshotOwner(snapshot, user)) {
    // Snapshot owner is allowed to delete the snapshot
    AuthResult result = AuthResult.allow("deleteSnapshot " + snapshot.getName(),
        "Snapshot owner check allowed", user, null, null, null);
    AccessChecker.logResult(result);
  } else {
    accessChecker.requirePermission(user, "deleteSnapshot " + snapshot.getName(), null,
      Action.ADMIN);
  }
}
 
Example 18
Source Project: hbase   Source File: FileArchiverNotifierImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * For the given snapshot, find all files which this {@code snapshotName} references. After a file
 * is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
 * {@code snapshotSizeChanges} is updated in concert.
 *
 * @param snapshotName The snapshot to check
 * @param filesToUpdate A mapping of archived files to their size
 * @param snapshotSizeChanges A mapping of snapshots and their change in size
 */
void bucketFilesToSnapshot(
    String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
        throws IOException {
  // A quick check to avoid doing work if the caller unnecessarily invoked this method.
  if (filesToUpdate.isEmpty()) {
    return;
  }

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        Long valueOrNull = filesToUpdate.remove(sf.getName());
        if (valueOrNull != null) {
          // This storefile was recently archived, we should update this snapshot with its size
          snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
        }
        // Short-circuit, if we have no more files that were archived, we don't need to iterate
        // over the rest of the snapshot.
        if (filesToUpdate.isEmpty()) {
          return;
        }
      }
    }
  }
}
 
Example 19
Source Project: hbase   Source File: RestoreSnapshotProcedure.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Action before any real action of restoring from snapshot.
 * @param env MasterProcedureEnv
 * @throws IOException
 */
private void prepareRestore(final MasterProcedureEnv env) throws IOException {
  final TableName tableName = getTableName();
  // Checks whether the table exists
  if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
    throw new TableNotFoundException(tableName);
  }

  // Check whether table is disabled.
  env.getMasterServices().checkTableModifiable(tableName);

  // Check that we have at least 1 CF
  if (modifiedTableDescriptor.getColumnFamilyCount() == 0) {
    throw new DoNotRetryIOException("Table " + getTableName().toString() +
      " should have at least one column family.");
  }

  if (!getTableName().isSystemTable()) {
    // Table already exist. Check and update the region quota for this table namespace.
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterConfiguration(),
      mfs.getFileSystem(),
      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
      snapshot);
    int snapshotRegionCount = manifest.getRegionManifestsMap().size();
    int tableRegionCount =
        ProcedureSyncWait.getMasterQuotaManager(env).getRegionCountOfTable(tableName);

    if (snapshotRegionCount > 0 && tableRegionCount != snapshotRegionCount) {
      ProcedureSyncWait.getMasterQuotaManager(env).checkAndUpdateNamespaceRegionQuota(
        tableName, snapshotRegionCount);
    }
  }
}
 
Example 20
Source Project: hbase   Source File: RestoreSnapshotProcedure.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Execute the on-disk Restore
 * @param env MasterProcedureEnv
 * @throws IOException
 **/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
  MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
  FileSystem fs = fileSystemManager.getFileSystem();
  Path rootDir = fileSystemManager.getRootDir();
  final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();

  LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
  try {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
    RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
      env.getMasterServices().getConfiguration(),
      fs,
      manifest,
      modifiedTableDescriptor,
      rootDir,
      monitorException,
      getMonitorStatus());

    RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
    regionsToRestore = metaChanges.getRegionsToRestore();
    regionsToRemove = metaChanges.getRegionsToRemove();
    regionsToAdd = metaChanges.getRegionsToAdd();
    parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap();
  } catch (IOException e) {
    String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
      + " failed in on-disk restore. Try re-running the restore command.";
    LOG.error(msg, e);
    monitorException.receive(
      new ForeignException(env.getMasterServices().getServerName().toString(), e));
    throw new IOException(msg, e);
  }
}
 
Example 21
Source Project: hbase   Source File: RestoreSnapshotProcedure.java    License: Apache License 2.0 5 votes vote down vote up
private void restoreSnapshotAcl(final MasterProcedureEnv env) throws IOException {
  if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
      && SnapshotDescriptionUtils
          .isSecurityAvailable(env.getMasterServices().getConfiguration())) {
    // restore acl of snapshot to table.
    RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, TableName.valueOf(snapshot.getTable()),
      env.getMasterServices().getConfiguration());
  }
}
 
Example 22
Source Project: hbase   Source File: CloneSnapshotProcedure.java    License: Apache License 2.0 5 votes vote down vote up
private void restoreSnapshotAcl(MasterProcedureEnv env) throws IOException {
  Configuration conf = env.getMasterServices().getConfiguration();
  if (restoreAcl && snapshot.hasUsersAndPermissions() && snapshot.getUsersAndPermissions() != null
      && SnapshotDescriptionUtils.isSecurityAvailable(conf)) {
    RestoreSnapshotHelper.restoreSnapshotAcl(snapshot, tableDescriptor.getTableName(), conf);
  }
}
 
Example 23
Source Project: hbase   Source File: SnapshotFileCache.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a snapshot file cache for all snapshots under the specified [root]/.snapshot on the
 * filesystem
 * @param fs {@link FileSystem} where the snapshots are stored
 * @param rootDir hbase root directory
 * @param workingFs {@link FileSystem} where ongoing snapshot mainifest files are stored
 * @param workingDir Location to store ongoing snapshot manifest files
 * @param cacheRefreshPeriod period (ms) with which the cache should be refreshed
 * @param cacheRefreshDelay amount of time to wait for the cache to be refreshed
 * @param refreshThreadName name of the cache refresh thread
 * @param inspectSnapshotFiles Filter to apply to each snapshot to extract the files.
 */
public SnapshotFileCache(FileSystem fs, Path rootDir, FileSystem workingFs, Path workingDir,
  long cacheRefreshPeriod, long cacheRefreshDelay, String refreshThreadName,
  SnapshotFileInspector inspectSnapshotFiles) {
  this.fs = fs;
  this.workingFs = workingFs;
  this.workingSnapshotDir = workingDir;
  this.fileInspector = inspectSnapshotFiles;
  this.snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  // periodically refresh the file cache to make sure we aren't superfluously saving files.
  this.refreshTimer = new Timer(refreshThreadName, true);
  this.refreshTimer.scheduleAtFixedRate(new RefreshCacheTask(), cacheRefreshDelay,
    cacheRefreshPeriod);
}
 
Example 24
Source Project: hbase   Source File: MasterSnapshotVerifier.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check that the snapshot description written in the filesystem matches the current snapshot
 * @param snapshotDir snapshot directory to check
 */
private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
  SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(workingDirFs,
      snapshotDir);
  if (!this.snapshot.equals(found)) {
    throw new CorruptedSnapshotException(
        "Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").",
        ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 
Example 25
Source Project: hbase   Source File: SnapshotManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Cleans up any snapshots in the snapshot/.tmp directory that were left from failed
 * snapshot attempts.
 *
 * @throws IOException if we can't reach the filesystem
 */
private void resetTempDir() throws IOException {
  // cleanup any existing snapshots.
  Path tmpdir = SnapshotDescriptionUtils.getWorkingSnapshotDir(rootDir,
      master.getConfiguration());
  FileSystem tmpFs = tmpdir.getFileSystem(master.getConfiguration());
  if (!tmpFs.delete(tmpdir, true)) {
    LOG.warn("Couldn't delete working snapshot directory: " + tmpdir);
  }
}
 
Example 26
Source Project: hbase   Source File: SnapshotManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Delete the specified snapshot
 * @param snapshot
 * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
 * @throws IOException For filesystem IOExceptions
 */
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
  // check to see if it is completed
  if (!isSnapshotCompleted(snapshot)) {
    throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
  }

  String snapshotName = snapshot.getName();
  // first create the snapshot description and check to see if it exists
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
  // just the "name" and it does not contains the "real" snapshot information
  snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  // call coproc pre hook
  MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
  org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
  if (cpHost != null) {
    snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
    cpHost.preDeleteSnapshot(snapshotPOJO);
  }

  LOG.debug("Deleting snapshot: " + snapshotName);
  // delete the existing snapshot
  if (!fs.delete(snapshotDir, true)) {
    throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
  }

  // call coproc post hook
  if (cpHost != null) {
    cpHost.postDeleteSnapshot(snapshotPOJO);
  }

}
 
Example 27
Source Project: hbase   Source File: SnapshotManager.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check to see if the snapshot is one of the currently completed snapshots
 * Returns true if the snapshot exists in the "completed snapshots folder".
 *
 * @param snapshot expected snapshot to check
 * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
 *         not stored
 * @throws IOException if the filesystem throws an unexpected exception,
 * @throws IllegalArgumentException if snapshot name is invalid.
 */
private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
  try {
    final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    // check to see if the snapshot already exists
    return fs.exists(snapshotDir);
  } catch (IllegalArgumentException iae) {
    throw new UnknownSnapshotException("Unexpected exception thrown", iae);
  }
}
 
Example 28
Source Project: hbase   Source File: SnapshotManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void initialize(MasterServices master, MetricsMaster metricsMaster) throws KeeperException,
    IOException, UnsupportedOperationException {
  this.master = master;

  this.rootDir = master.getMasterFileSystem().getRootDir();
  checkSnapshotSupport(master.getConfiguration(), master.getMasterFileSystem());

  // get the configuration for the coordinator
  Configuration conf = master.getConfiguration();
  long wakeFrequency = conf.getInt(SNAPSHOT_WAKE_MILLIS_KEY, SNAPSHOT_WAKE_MILLIS_DEFAULT);
  long timeoutMillis = Math.max(
          conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
                  SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME),
          conf.getLong(SnapshotDescriptionUtils.MASTER_SNAPSHOT_TIMEOUT_MILLIS,
                  SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME));
  int opThreads = conf.getInt(SNAPSHOT_POOL_THREADS_KEY, SNAPSHOT_POOL_THREADS_DEFAULT);

  // setup the default procedure coordinator
  String name = master.getServerName().toString();
  ThreadPoolExecutor tpool = ProcedureCoordinator.defaultPool(name, opThreads);
  ProcedureCoordinatorRpcs comms = new ZKProcedureCoordinator(
      master.getZooKeeper(), SnapshotManager.ONLINE_SNAPSHOT_CONTROLLER_DESCRIPTION, name);

  this.coordinator = new ProcedureCoordinator(comms, tpool, timeoutMillis, wakeFrequency);
  this.executorService = master.getExecutorService();
  resetTempDir();
  snapshotHandlerChoreCleanerTask =
      scheduleThreadPool.scheduleAtFixedRate(this::cleanupSentinels, 10, 10, TimeUnit.SECONDS);
}
 
Example 29
Source Project: hbase   Source File: TestSnapshotFileCache.java    License: Apache License 2.0 5 votes vote down vote up
protected static void initCommon() throws Exception {
  UTIL.startMiniDFSCluster(1);
  fs = UTIL.getDFSCluster().getFileSystem();
  rootDir = UTIL.getDefaultRootDirPath();
  snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
  conf = UTIL.getConfiguration();
}
 
Example 30
Source Project: hbase   Source File: TestSnapshotHFileCleaner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFindsSnapshotFilesWhenCleaning() throws IOException {
  CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);

  FileSystem fs = FileSystem.get(conf);
  SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
  cleaner.setConf(conf);

  // write an hfile to the snapshot directory
  String snapshotName = "snapshot";
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  RegionInfo mockRegion = RegionInfoBuilder.newBuilder(tableName).build();
  Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
  Path familyDir = new Path(regionSnapshotDir, "family");
  // create a reference to a supposedly valid hfile
  String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
  Path refFile = new Path(familyDir, hfile);

  // make sure the reference file exists
  fs.create(refFile);

  // create the hfile in the archive
  fs.mkdirs(archivedHfileDir);
  fs.createNewFile(new Path(archivedHfileDir, hfile));

  // make sure that the file isn't deletable
  assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
}