Java Code Examples for org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils#readSnapshotInfo()

/**
 * get region infos
 *
 * @param zookeeperQuorum     zookeeper quorum
 * @param zookeeperClientPort zookeeper client port
 * @param hBaseRootDir        HBase root dir
 * @param snapshotName        snapshot name
 * @return region info list
 * @throws IOException IOException
 */
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
                                               String hBaseRootDir, String snapshotName) throws IOException {
    try {
        Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
        Path root = new Path(hBaseRootDir);
        FileSystem fs = FileSystem.get(conf);
        Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
        HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
        return Utils.getRegionInfosFromManifest(manifest);
    } catch (IOException ex) {
        logger.error("get region info error: " + ex.getMessage(), ex);
        throw ex;
    }
}
 

/**
 * Get table descriptor
 * @param tableName is the table backed up
 * @return {@link TableDescriptor} saved in backup image of the table
 */
TableDescriptor getTableDesc(TableName tableName) throws IOException {
  Path tableInfoPath = this.getTableInfoPath(tableName);
  SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
  TableDescriptor tableDescriptor = manifest.getTableDescriptor();
  if (!tableDescriptor.getTableName().equals(tableName)) {
    LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
            + tableInfoPath.toString());
    LOG.error("tableDescriptor.getNameAsString() = "
            + tableDescriptor.getTableName().getNameAsString());
    throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
        + " under tableInfoPath: " + tableInfoPath.toString());
  }
  return tableDescriptor;
}
 

private void openWithoutRestoringSnapshot() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotProtos.SnapshotDescription snapshotDesc =
      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
  }

  regions = new ArrayList<>(regionManifests.size());
  regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
    .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
  htd = manifest.getTableDescriptor();
}
 

private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 

@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
	String snapshotName;
	Configuration conf = context.getConnection().getQueryServices().getConfiguration();
	if((snapshotName = getSnapshotName(conf)) != null) {
		try {
			Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
			FileSystem fs = rootDir.getFileSystem(conf);
			Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
			SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
			SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
			return getRegionLocationsFromManifest(manifest);
		}
		catch (Exception e) {
			throw new RuntimeException(e);
		}
	}
	else {
		return context.getConnection().getQueryServices().getAllTableRegions(tableName);
	}
}
 

/**
 * For the given snapshot, find all files which this {@code snapshotName} references. After a file
 * is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
 * {@code snapshotSizeChanges} is updated in concert.
 *
 * @param snapshotName The snapshot to check
 * @param filesToUpdate A mapping of archived files to their size
 * @param snapshotSizeChanges A mapping of snapshots and their change in size
 */
void bucketFilesToSnapshot(
    String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
        throws IOException {
  // A quick check to avoid doing work if the caller unnecessarily invoked this method.
  if (filesToUpdate.isEmpty()) {
    return;
  }

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        Long valueOrNull = filesToUpdate.remove(sf.getName());
        if (valueOrNull != null) {
          // This storefile was recently archived, we should update this snapshot with its size
          snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
        }
        // Short-circuit, if we have no more files that were archived, we don't need to iterate
        // over the rest of the snapshot.
        if (filesToUpdate.isEmpty()) {
          return;
        }
      }
    }
  }
}
 

/**
 * Check that the snapshot description written in the filesystem matches the current snapshot
 * @param snapshotDir snapshot directory to check
 */
private void verifySnapshotDescription(Path snapshotDir) throws CorruptedSnapshotException {
  SnapshotDescription found = SnapshotDescriptionUtils.readSnapshotInfo(workingDirFs,
      snapshotDir);
  if (!this.snapshot.equals(found)) {
    throw new CorruptedSnapshotException(
        "Snapshot read (" + found + ") doesn't equal snapshot we ran (" + snapshot + ").",
        ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 

/**
 * Delete the specified snapshot
 * @param snapshot
 * @throws SnapshotDoesNotExistException If the specified snapshot does not exist.
 * @throws IOException For filesystem IOExceptions
 */
public void deleteSnapshot(SnapshotDescription snapshot) throws IOException {
  // check to see if it is completed
  if (!isSnapshotCompleted(snapshot)) {
    throw new SnapshotDoesNotExistException(ProtobufUtil.createSnapshotDesc(snapshot));
  }

  String snapshotName = snapshot.getName();
  // first create the snapshot description and check to see if it exists
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  // Get snapshot info from file system. The one passed as parameter is a "fake" snapshotInfo with
  // just the "name" and it does not contains the "real" snapshot information
  snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  // call coproc pre hook
  MasterCoprocessorHost cpHost = master.getMasterCoprocessorHost();
  org.apache.hadoop.hbase.client.SnapshotDescription snapshotPOJO = null;
  if (cpHost != null) {
    snapshotPOJO = ProtobufUtil.createSnapshotDesc(snapshot);
    cpHost.preDeleteSnapshot(snapshotPOJO);
  }

  LOG.debug("Deleting snapshot: " + snapshotName);
  // delete the existing snapshot
  if (!fs.delete(snapshotDir, true)) {
    throw new HBaseSnapshotException("Failed to delete snapshot directory: " + snapshotDir);
  }

  // call coproc post hook
  if (cpHost != null) {
    cpHost.postDeleteSnapshot(snapshotPOJO);
  }

}
 

private void createAndRestoreTable(Connection conn, TableName tableName, TableName newTableName,
    Path tableBackupPath, boolean truncateIfExists, String lastIncrBackupId) throws IOException {
  if (newTableName == null) {
    newTableName = tableName;
  }
  FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);

  // get table descriptor first
  TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId);
  if (tableDescriptor != null) {
    LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId);
  }

  if (tableDescriptor == null) {
    Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
    if (fileSys.exists(tableSnapshotPath)) {
      // snapshot path exist means the backup path is in HDFS
      // check whether snapshot dir already recorded for target table
      if (snapshotMap.get(tableName) != null) {
        SnapshotDescription desc =
            SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
        tableDescriptor = manifest.getTableDescriptor();
      } else {
        tableDescriptor = getTableDesc(tableName);
        snapshotMap.put(tableName, getTableInfoPath(tableName));
      }
      if (tableDescriptor == null) {
        LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
      }
    } else {
      throw new IOException("Table snapshot directory: " +
          tableSnapshotPath + " does not exist.");
    }
  }

  Path tableArchivePath = getTableArchivePath(tableName);
  if (tableArchivePath == null) {
    if (tableDescriptor != null) {
      // find table descriptor but no archive dir means the table is empty, create table and exit
      if (LOG.isDebugEnabled()) {
        LOG.debug("find table descriptor but no archive dir for table " + tableName
            + ", will only create table");
      }
      tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
      checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor,
        truncateIfExists);
      return;
    } else {
      throw new IllegalStateException("Cannot restore hbase table because directory '"
          + " tableArchivePath is null.");
    }
  }

  if (tableDescriptor == null) {
    tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build();
  } else {
    tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
  }

  // record all region dirs:
  // load all files in dir
  try {
    ArrayList<Path> regionPathList = getRegionList(tableName);

    // should only try to create the table with all region informations, so we could pre-split
    // the regions in fine grain
    checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, regionPathList,
      tableDescriptor, truncateIfExists);
    RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf);
    Path[] paths = new Path[regionPathList.size()];
    regionPathList.toArray(paths);
    restoreService.run(paths, new TableName[]{tableName}, new TableName[] {newTableName}, true);

  } catch (Exception e) {
    LOG.error(e.toString(), e);
    throw new IllegalStateException("Cannot restore hbase table", e);
  }
}
 

/**
 * Restore or Clone the specified snapshot
 * @param reqSnapshot
 * @param nonceKey unique identifier to prevent duplicated RPC
 * @throws IOException
 */
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
    final boolean restoreAcl) throws IOException {
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);

  // check if the snapshot exists
  if (!fs.exists(snapshotDir)) {
    LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
    throw new SnapshotDoesNotExistException(
      ProtobufUtil.createSnapshotDesc(reqSnapshot));
  }

  // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
  // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
  // information.
  SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
      snapshotDir, snapshot);
  TableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
  TableName tableName = TableName.valueOf(reqSnapshot.getTable());

  // sanity check the new table descriptor
  TableDescriptorChecker.sanityCheck(master.getConfiguration(), snapshotTableDesc);

  // stop tracking "abandoned" handlers
  cleanupSentinels();

  // Verify snapshot validity
  SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);

  // Execute the restore/clone operation
  long procId;
  if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
    procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
      restoreAcl);
  } else {
    procId =
        cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
  }
  return procId;
}
 

public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName,
    Path rootDir, FileSystem fs) throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}