Java Code Examples for org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils#getCompletedSnapshotDir()

/**
 * get region infos
 *
 * @param zookeeperQuorum     zookeeper quorum
 * @param zookeeperClientPort zookeeper client port
 * @param hBaseRootDir        HBase root dir
 * @param snapshotName        snapshot name
 * @return region info list
 * @throws IOException IOException
 */
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
                                               String hBaseRootDir, String snapshotName) throws IOException {
    try {
        Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
        Path root = new Path(hBaseRootDir);
        FileSystem fs = FileSystem.get(conf);
        Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
        HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
        return Utils.getRegionInfosFromManifest(manifest);
    } catch (IOException ex) {
        logger.error("get region info error: " + ex.getMessage(), ex);
        throw ex;
    }
}
 

private void openWithoutRestoringSnapshot() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotProtos.SnapshotDescription snapshotDesc =
      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
  }

  regions = new ArrayList<>(regionManifests.size());
  regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
    .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
  htd = manifest.getTableDescriptor();
}
 

private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 

@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
	String snapshotName;
	Configuration conf = context.getConnection().getQueryServices().getConfiguration();
	if((snapshotName = getSnapshotName(conf)) != null) {
		try {
			Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
			FileSystem fs = rootDir.getFileSystem(conf);
			Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
			SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
			SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
			return getRegionLocationsFromManifest(manifest);
		}
		catch (Exception e) {
			throw new RuntimeException(e);
		}
	}
	else {
		return context.getConnection().getQueryServices().getAllTableRegions(tableName);
	}
}
 

/**
 * For the given snapshot, find all files which this {@code snapshotName} references. After a file
 * is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
 * {@code snapshotSizeChanges} is updated in concert.
 *
 * @param snapshotName The snapshot to check
 * @param filesToUpdate A mapping of archived files to their size
 * @param snapshotSizeChanges A mapping of snapshots and their change in size
 */
void bucketFilesToSnapshot(
    String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
        throws IOException {
  // A quick check to avoid doing work if the caller unnecessarily invoked this method.
  if (filesToUpdate.isEmpty()) {
    return;
  }

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        Long valueOrNull = filesToUpdate.remove(sf.getName());
        if (valueOrNull != null) {
          // This storefile was recently archived, we should update this snapshot with its size
          snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
        }
        // Short-circuit, if we have no more files that were archived, we don't need to iterate
        // over the rest of the snapshot.
        if (filesToUpdate.isEmpty()) {
          return;
        }
      }
    }
  }
}
 

/**
 * Execute the on-disk Restore
 * @param env MasterProcedureEnv
 * @throws IOException
 **/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
  MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
  FileSystem fs = fileSystemManager.getFileSystem();
  Path rootDir = fileSystemManager.getRootDir();
  final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();

  LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
  try {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
    RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
      env.getMasterServices().getConfiguration(),
      fs,
      manifest,
      modifiedTableDescriptor,
      rootDir,
      monitorException,
      getMonitorStatus());

    RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
    regionsToRestore = metaChanges.getRegionsToRestore();
    regionsToRemove = metaChanges.getRegionsToRemove();
    regionsToAdd = metaChanges.getRegionsToAdd();
    parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap();
  } catch (IOException e) {
    String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
      + " failed in on-disk restore. Try re-running the restore command.";
    LOG.error(msg, e);
    monitorException.receive(
      new ForeignException(env.getMasterServices().getServerName().toString(), e));
    throw new IOException(msg, e);
  }
}
 

/**
 * Delete the specified snapshot
 * @param snapshot
 * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
 * @throws IOException For filesystem IOExceptions
 */
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
  // check to see if it is completed
  if (!isSnapshotCompleted(snapshot)) {
    throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
  }

  String snapshotName = snapshot.getName();
  // first create the snapshot description and check to see if it exists
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
  // just the "name" and it does not contains the "real" snapshot information
  snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  // call coproc pre hook
  MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
  org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
  if (cpHost != null) {
    snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
    cpHost.preDeleteSnapshot(snapshotPOJO);
  }

  LOG.debug("Deleting snapshot: " + snapshotName);
  // delete the existing snapshot
  if (!fs.delete(snapshotDir, true)) {
    throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
  }

  // call coproc post hook
  if (cpHost != null) {
    cpHost.postDeleteSnapshot(snapshotPOJO);
  }

}
 

/**
 * Check to see if the snapshot is one of the currently completed snapshots
 * Returns true if the snapshot exists in the "completed snapshots folder".
 *
 * @param snapshot expected snapshot to check
 * @return <tt>true</tt> if the snapshot is stored on the {@link FileSystem}, <tt>false</tt> if is
 *         not stored
 * @throws IOException if the filesystem throws an unexpected exception,
 * @throws IllegalArgumentException if snapshot name is invalid.
 */
private boolean isSnapshotCompleted(SnapshotDescription snapshot) throws IOException {
  try {
    final Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    FileSystem fs = master.getMasterFileSystem().getFileSystem();
    // check to see if the snapshot already exists
    return fs.exists(snapshotDir);
  } catch (IllegalArgumentException iae) {
    throw new UnknownSnapshotException("Unexpected exception thrown", iae);
  }
}
 

@Test
public void testFindsSnapshotFilesWhenCleaning() throws IOException {
  CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY);

  FileSystem fs = FileSystem.get(conf);
  SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner();
  cleaner.setConf(conf);

  // write an hfile to the snapshot directory
  String snapshotName = "snapshot";
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  RegionInfo mockRegion = RegionInfoBuilder.newBuilder(tableName).build();
  Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName());
  Path familyDir = new Path(regionSnapshotDir, "family");
  // create a reference to a supposedly valid hfile
  String hfile = "fd1e73e8a96c486090c5cec07b4894c4";
  Path refFile = new Path(familyDir, hfile);

  // make sure the reference file exists
  fs.create(refFile);

  // create the hfile in the archive
  fs.mkdirs(archivedHfileDir);
  fs.createNewFile(new Path(archivedHfileDir, hfile));

  // make sure that the file isn't deletable
  assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile)));
}
 

/**
 * Restore or Clone the specified snapshot
 * @param reqSnapshot
 * @param nonceKey unique identifier to prevent duplicated RPC
 * @throws IOException
 */
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
    final boolean restoreAcl) throws IOException {
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);

  // check if the snapshot exists
  if (!fs.exists(snapshotDir)) {
    LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
    throw new SnapshotDoesNotExistException(
      ProtobufUtil.createSnapshotDesc(reqSnapshot));
  }

  // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
  // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
  // information.
  SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
      snapshotDir, snapshot);
  TableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
  TableName tableName = TableName.valueOf(reqSnapshot.getTable());

  // sanity check the new table descriptor
  TableDescriptorChecker.sanityCheck(master.getConfiguration(), snapshotTableDesc);

  // stop tracking "abandoned" handlers
  cleanupSentinels();

  // Verify snapshot validity
  SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);

  // Execute the restore/clone operation
  long procId;
  if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
    procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
      restoreAcl);
  } else {
    procId =
        cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
  }
  return procId;
}
 

public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName,
    Path rootDir, FileSystem fs) throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}