Java Code Examples for org.apache.hadoop.hbase.snapshot.SnapshotManifest

The following examples show how to use org.apache.hadoop.hbase.snapshot.SnapshotManifest. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto-hbase-connector   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
public static List<HRegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) {
    List<SnapshotProtos.SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
    if (regionManifests == null) {
        throw new IllegalArgumentException("Snapshot seems empty");
    }

    List<HRegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size());

    for (SnapshotProtos.SnapshotRegionManifest regionManifest : regionManifests) {
        HRegionInfo hri = HRegionInfo.convert(regionManifest.getRegionInfo());
        if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
            continue;
        }
        regionInfos.add(hri);
    }
    return regionInfos;
}
 
Example 2
Source Project: presto-hbase-connector   Source File: Utils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * get region infos
 *
 * @param zookeeperQuorum     zookeeper quorum
 * @param zookeeperClientPort zookeeper client port
 * @param hBaseRootDir        HBase root dir
 * @param snapshotName        snapshot name
 * @return region info list
 * @throws IOException IOException
 */
public static List<HRegionInfo> getRegionInfos(String zookeeperQuorum, String zookeeperClientPort,
                                               String hBaseRootDir, String snapshotName) throws IOException {
    try {
        Configuration conf = Utils.getHadoopConf(zookeeperQuorum, zookeeperClientPort);
        Path root = new Path(hBaseRootDir);
        FileSystem fs = FileSystem.get(conf);
        Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, root);
        HBaseProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
        return Utils.getRegionInfosFromManifest(manifest);
    } catch (IOException ex) {
        logger.error("get region info error: " + ex.getMessage(), ex);
        throw ex;
    }
}
 
Example 3
Source Project: hbase   Source File: RestoreTool.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get table descriptor
 * @param tableName is the table backed up
 * @return {@link TableDescriptor} saved in backup image of the table
 */
TableDescriptor getTableDesc(TableName tableName) throws IOException {
  Path tableInfoPath = this.getTableInfoPath(tableName);
  SnapshotDescription desc = SnapshotDescriptionUtils.readSnapshotInfo(fs, tableInfoPath);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, tableInfoPath, desc);
  TableDescriptor tableDescriptor = manifest.getTableDescriptor();
  if (!tableDescriptor.getTableName().equals(tableName)) {
    LOG.error("couldn't find Table Desc for table: " + tableName + " under tableInfoPath: "
            + tableInfoPath.toString());
    LOG.error("tableDescriptor.getNameAsString() = "
            + tableDescriptor.getTableName().getNameAsString());
    throw new FileNotFoundException("couldn't find Table Desc for table: " + tableName
        + " under tableInfoPath: " + tableInfoPath.toString());
  }
  return tableDescriptor;
}
 
Example 4
Source Project: hbase   Source File: TableSnapshotScanner.java    License: Apache License 2.0 6 votes vote down vote up
private void openWithoutRestoringSnapshot() throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotProtos.SnapshotDescription snapshotDesc =
      SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);

  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty, snapshotName: " + snapshotName);
  }

  regions = new ArrayList<>(regionManifests.size());
  regionManifests.stream().map(r -> ProtobufUtil.toRegionInfo(r.getRegionInfo()))
    .filter(this::isValidRegion).sorted().forEach(r -> regions.add(r));
  htd = manifest.getTableDescriptor();
}
 
Example 5
Source Project: hbase   Source File: CloneSnapshotProcedure.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Action before cloning from snapshot.
 * @param env MasterProcedureEnv
 * @throws IOException
 * @throws InterruptedException
 */
private void preCloneSnapshot(final MasterProcedureEnv env)
    throws IOException, InterruptedException {
  if (!getTableName().isSystemTable()) {
    // Check and update namespace quota
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();

    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterConfiguration(),
      mfs.getFileSystem(),
      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
      snapshot);

    ProcedureSyncWait.getMasterQuotaManager(env)
      .checkNamespaceTableAndRegionQuota(getTableName(), manifest.getRegionManifestsMap().size());
  }

  final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
  if (cpHost != null) {
    cpHost.preCreateTableAction(tableDescriptor, null, getUser());
  }
}
 
Example 6
Source Project: hbase   Source File: TestFileArchiverNotifierImpl.java    License: Apache License 2.0 6 votes vote down vote up
private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
  HashSet<String> files = new HashSet<>();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
      fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        files.add(sf.getName());
      }
    }
  }
  return files;
}
 
Example 7
Source Project: hbase   Source File: TableSnapshotInputFormatImpl.java    License: Apache License 2.0 6 votes vote down vote up
public static List<RegionInfo> getRegionInfosFromManifest(SnapshotManifest manifest) {
  List<SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
  if (regionManifests == null) {
    throw new IllegalArgumentException("Snapshot seems empty");
  }

  List<RegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size());

  for (SnapshotRegionManifest regionManifest : regionManifests) {
    RegionInfo hri = ProtobufUtil.toRegionInfo(regionManifest.getRegionInfo());
    if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
      continue;
    }
    regionInfos.add(hri);
  }
  return regionInfos;
}
 
Example 8
Source Project: phoenix   Source File: MapReduceParallelScanGrouper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<HRegionLocation> getRegionBoundaries(StatementContext context, byte[] tableName) throws SQLException {
	String snapshotName;
	Configuration conf = context.getConnection().getQueryServices().getConfiguration();
	if((snapshotName = getSnapshotName(conf)) != null) {
		try {
			Path rootDir = new Path(conf.get(HConstants.HBASE_DIR));
			FileSystem fs = rootDir.getFileSystem(conf);
			Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
			SnapshotDescription snapshotDescription = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
			SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDescription);
			return getRegionLocationsFromManifest(manifest);
		}
		catch (Exception e) {
			throw new RuntimeException(e);
		}
	}
	else {
		return context.getConnection().getQueryServices().getAllTableRegions(tableName);
	}
}
 
Example 9
Source Project: hbase   Source File: FileArchiverNotifierImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * For the given snapshot, find all files which this {@code snapshotName} references. After a file
 * is found to be referenced by the snapshot, it is removed from {@code filesToUpdate} and
 * {@code snapshotSizeChanges} is updated in concert.
 *
 * @param snapshotName The snapshot to check
 * @param filesToUpdate A mapping of archived files to their size
 * @param snapshotSizeChanges A mapping of snapshots and their change in size
 */
void bucketFilesToSnapshot(
    String snapshotName, Map<String,Long> filesToUpdate, Map<String,Long> snapshotSizeChanges)
        throws IOException {
  // A quick check to avoid doing work if the caller unnecessarily invoked this method.
  if (filesToUpdate.isEmpty()) {
    return;
  }

  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
      snapshotName, CommonFSUtils.getRootDir(conf));
  SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        Long valueOrNull = filesToUpdate.remove(sf.getName());
        if (valueOrNull != null) {
          // This storefile was recently archived, we should update this snapshot with its size
          snapshotSizeChanges.merge(snapshotName, valueOrNull, Long::sum);
        }
        // Short-circuit, if we have no more files that were archived, we don't need to iterate
        // over the rest of the snapshot.
        if (filesToUpdate.isEmpty()) {
          return;
        }
      }
    }
  }
}
 
Example 10
Source Project: hbase   Source File: FileArchiverNotifierImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Extracts the names of the store files referenced by this snapshot which satisfy the given
 * predicate (the predicate returns {@code true}).
 */
Set<StoreFileReference> getStoreFilesFromSnapshot(
    SnapshotManifest manifest, Predicate<String> filter) {
  Set<StoreFileReference> references = new HashSet<>();
  // For each region referenced by the snapshot
  for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
    StoreFileReference regionReference = new StoreFileReference(
        ProtobufUtil.toRegionInfo(rm.getRegionInfo()).getEncodedName());

    // For each column family in this region
    for (FamilyFiles ff : rm.getFamilyFilesList()) {
      final String familyName = ff.getFamilyName().toStringUtf8();
      // And each store file in that family
      for (StoreFile sf : ff.getStoreFilesList()) {
        String storeFileName = sf.getName();
        // A snapshot only "inherits" a files size if it uniquely refers to it (no table
        // and no other snapshot references it).
        if (filter.test(storeFileName)) {
          regionReference.addFamilyStoreFile(familyName, storeFileName);
        }
      }
    }
    // Only add this Region reference if we retained any files.
    if (!regionReference.getFamilyToFilesMapping().isEmpty()) {
      references.add(regionReference);
    }
  }
  return references;
}
 
Example 11
Source Project: hbase   Source File: RestoreSnapshotProcedure.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Action before any real action of restoring from snapshot.
 * @param env MasterProcedureEnv
 * @throws IOException
 */
private void prepareRestore(final MasterProcedureEnv env) throws IOException {
  final TableName tableName = getTableName();
  // Checks whether the table exists
  if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
    throw new TableNotFoundException(tableName);
  }

  // Check whether table is disabled.
  env.getMasterServices().checkTableModifiable(tableName);

  // Check that we have at least 1 CF
  if (modifiedTableDescriptor.getColumnFamilyCount() == 0) {
    throw new DoNotRetryIOException("Table " + getTableName().toString() +
      " should have at least one column family.");
  }

  if (!getTableName().isSystemTable()) {
    // Table already exist. Check and update the region quota for this table namespace.
    final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterConfiguration(),
      mfs.getFileSystem(),
      SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, mfs.getRootDir()),
      snapshot);
    int snapshotRegionCount = manifest.getRegionManifestsMap().size();
    int tableRegionCount =
        ProcedureSyncWait.getMasterQuotaManager(env).getRegionCountOfTable(tableName);

    if (snapshotRegionCount > 0 && tableRegionCount != snapshotRegionCount) {
      ProcedureSyncWait.getMasterQuotaManager(env).checkAndUpdateNamespaceRegionQuota(
        tableName, snapshotRegionCount);
    }
  }
}
 
Example 12
Source Project: hbase   Source File: RestoreSnapshotProcedure.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Execute the on-disk Restore
 * @param env MasterProcedureEnv
 * @throws IOException
 **/
private void restoreSnapshot(final MasterProcedureEnv env) throws IOException {
  MasterFileSystem fileSystemManager = env.getMasterServices().getMasterFileSystem();
  FileSystem fs = fileSystemManager.getFileSystem();
  Path rootDir = fileSystemManager.getRootDir();
  final ForeignExceptionDispatcher monitorException = new ForeignExceptionDispatcher();

  LOG.info("Starting restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot));
  try {
    Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshot, rootDir);
    SnapshotManifest manifest = SnapshotManifest.open(
      env.getMasterServices().getConfiguration(), fs, snapshotDir, snapshot);
    RestoreSnapshotHelper restoreHelper = new RestoreSnapshotHelper(
      env.getMasterServices().getConfiguration(),
      fs,
      manifest,
      modifiedTableDescriptor,
      rootDir,
      monitorException,
      getMonitorStatus());

    RestoreSnapshotHelper.RestoreMetaChanges metaChanges = restoreHelper.restoreHdfsRegions();
    regionsToRestore = metaChanges.getRegionsToRestore();
    regionsToRemove = metaChanges.getRegionsToRemove();
    regionsToAdd = metaChanges.getRegionsToAdd();
    parentsToChildrenPairMap = metaChanges.getParentToChildrenPairMap();
  } catch (IOException e) {
    String msg = "restore snapshot=" + ClientSnapshotDescriptionUtils.toString(snapshot)
      + " failed in on-disk restore. Try re-running the restore command.";
    LOG.error(msg, e);
    monitorException.receive(
      new ForeignException(env.getMasterServices().getServerName().toString(), e));
    throw new IOException(msg, e);
  }
}
 
Example 13
Source Project: hbase   Source File: MasterSnapshotVerifier.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verify that the snapshot in the directory is a valid snapshot
 * @param snapshotDir snapshot directory to check
 * @param snapshotServers {@link org.apache.hadoop.hbase.ServerName} of the servers
 *        that are involved in the snapshot
 * @throws CorruptedSnapshotException if the snapshot is invalid
 * @throws IOException if there is an unexpected connection issue to the filesystem
 */
public void verifySnapshot(Path snapshotDir, Set<String> snapshotServers)
    throws CorruptedSnapshotException, IOException {
  SnapshotManifest manifest = SnapshotManifest.open(services.getConfiguration(), workingDirFs,
                                                    snapshotDir, snapshot);
  // verify snapshot info matches
  verifySnapshotDescription(snapshotDir);

  // check that tableinfo is a valid table description
  verifyTableInfo(manifest);

  // check that each region is valid
  verifyRegions(manifest);
}
 
Example 14
Source Project: hbase   Source File: MasterSnapshotVerifier.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Check that the table descriptor for the snapshot is a valid table descriptor
 * @param manifest snapshot manifest to inspect
 */
private void verifyTableInfo(final SnapshotManifest manifest) throws IOException {
  TableDescriptor htd = manifest.getTableDescriptor();
  if (htd == null) {
    throw new CorruptedSnapshotException("Missing Table Descriptor",
      ProtobufUtil.createSnapshotDesc(snapshot));
  }

  if (!htd.getTableName().getNameAsString().equals(snapshot.getTable())) {
    throw new CorruptedSnapshotException(
        "Invalid Table Descriptor. Expected " + snapshot.getTable() + " name, got "
            + htd.getTableName().getNameAsString(), ProtobufUtil.createSnapshotDesc(snapshot));
  }
}
 
Example 15
Source Project: hbase   Source File: MultiTableSnapshotInputFormatImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the list of splits extracted from the scans/snapshots pushed to conf by
 * {@link
 * #setInput(org.apache.hadoop.conf.Configuration, java.util.Map, org.apache.hadoop.fs.Path)}
 *
 * @param conf Configuration to determine splits from
 * @return Return the list of splits extracted from the scans/snapshots pushed to conf
 * @throws IOException
 */
public List<TableSnapshotInputFormatImpl.InputSplit> getSplits(Configuration conf)
    throws IOException {
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = rootDir.getFileSystem(conf);

  List<TableSnapshotInputFormatImpl.InputSplit> rtn = Lists.newArrayList();

  Map<String, Collection<Scan>> snapshotsToScans = getSnapshotsToScans(conf);
  Map<String, Path> snapshotsToRestoreDirs = getSnapshotDirs(conf);
  for (Map.Entry<String, Collection<Scan>> entry : snapshotsToScans.entrySet()) {
    String snapshotName = entry.getKey();

    Path restoreDir = snapshotsToRestoreDirs.get(snapshotName);

    SnapshotManifest manifest =
        TableSnapshotInputFormatImpl.getSnapshotManifest(conf, snapshotName, rootDir, fs);
    List<RegionInfo> regionInfos =
        TableSnapshotInputFormatImpl.getRegionInfosFromManifest(manifest);

    for (Scan scan : entry.getValue()) {
      List<TableSnapshotInputFormatImpl.InputSplit> splits =
          TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf);
      rtn.addAll(splits);
    }
  }
  return rtn;
}
 
Example 16
Source Project: hbase   Source File: RestoreTool.java    License: Apache License 2.0 4 votes vote down vote up
private void createAndRestoreTable(Connection conn, TableName tableName, TableName newTableName,
    Path tableBackupPath, boolean truncateIfExists, String lastIncrBackupId) throws IOException {
  if (newTableName == null) {
    newTableName = tableName;
  }
  FileSystem fileSys = tableBackupPath.getFileSystem(this.conf);

  // get table descriptor first
  TableDescriptor tableDescriptor = getTableDescriptor(fileSys, tableName, lastIncrBackupId);
  if (tableDescriptor != null) {
    LOG.debug("Retrieved descriptor: " + tableDescriptor + " thru " + lastIncrBackupId);
  }

  if (tableDescriptor == null) {
    Path tableSnapshotPath = getTableSnapshotPath(backupRootPath, tableName, backupId);
    if (fileSys.exists(tableSnapshotPath)) {
      // snapshot path exist means the backup path is in HDFS
      // check whether snapshot dir already recorded for target table
      if (snapshotMap.get(tableName) != null) {
        SnapshotDescription desc =
            SnapshotDescriptionUtils.readSnapshotInfo(fileSys, tableSnapshotPath);
        SnapshotManifest manifest = SnapshotManifest.open(conf, fileSys, tableSnapshotPath, desc);
        tableDescriptor = manifest.getTableDescriptor();
      } else {
        tableDescriptor = getTableDesc(tableName);
        snapshotMap.put(tableName, getTableInfoPath(tableName));
      }
      if (tableDescriptor == null) {
        LOG.debug("Found no table descriptor in the snapshot dir, previous schema would be lost");
      }
    } else {
      throw new IOException("Table snapshot directory: " +
          tableSnapshotPath + " does not exist.");
    }
  }

  Path tableArchivePath = getTableArchivePath(tableName);
  if (tableArchivePath == null) {
    if (tableDescriptor != null) {
      // find table descriptor but no archive dir means the table is empty, create table and exit
      if (LOG.isDebugEnabled()) {
        LOG.debug("find table descriptor but no archive dir for table " + tableName
            + ", will only create table");
      }
      tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
      checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, null, tableDescriptor,
        truncateIfExists);
      return;
    } else {
      throw new IllegalStateException("Cannot restore hbase table because directory '"
          + " tableArchivePath is null.");
    }
  }

  if (tableDescriptor == null) {
    tableDescriptor = TableDescriptorBuilder.newBuilder(newTableName).build();
  } else {
    tableDescriptor = TableDescriptorBuilder.copy(newTableName, tableDescriptor);
  }

  // record all region dirs:
  // load all files in dir
  try {
    ArrayList<Path> regionPathList = getRegionList(tableName);

    // should only try to create the table with all region informations, so we could pre-split
    // the regions in fine grain
    checkAndCreateTable(conn, tableBackupPath, tableName, newTableName, regionPathList,
      tableDescriptor, truncateIfExists);
    RestoreJob restoreService = BackupRestoreFactory.getRestoreJob(conf);
    Path[] paths = new Path[regionPathList.size()];
    regionPathList.toArray(paths);
    restoreService.run(paths, new TableName[]{tableName}, new TableName[] {newTableName}, true);

  } catch (Exception e) {
    LOG.error(e.toString(), e);
    throw new IllegalStateException("Cannot restore hbase table", e);
  }
}
 
Example 17
Source Project: hbase   Source File: MasterSnapshotVerifier.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Check that all the regions in the snapshot are valid, and accounted for.
 * @param manifest snapshot manifest to inspect
 * @throws IOException if we can't reach hbase:meta or read the files from the FS
 */
private void verifyRegions(final SnapshotManifest manifest) throws IOException {
  List<RegionInfo> regions;
  if (TableName.META_TABLE_NAME.equals(tableName)) {
    regions = MetaTableLocator.getMetaRegions(services.getZooKeeper());
  } else {
    regions = MetaTableAccessor.getTableRegions(services.getConnection(), tableName);
  }
  // Remove the non-default regions
  RegionReplicaUtil.removeNonDefaultRegions(regions);

  Map<String, SnapshotRegionManifest> regionManifests = manifest.getRegionManifestsMap();
  if (regionManifests == null) {
    String msg = "Snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot) + " looks empty";
    LOG.error(msg);
    throw new CorruptedSnapshotException(msg);
  }

  String errorMsg = "";
  boolean hasMobStore = false;
  // the mob region is a dummy region, it's not a real region in HBase.
  // the mob region has a special name, it could be found by the region name.
  if (regionManifests.get(MobUtils.getMobRegionInfo(tableName).getEncodedName()) != null) {
    hasMobStore = true;
  }
  int realRegionCount = hasMobStore ? regionManifests.size() - 1 : regionManifests.size();
  if (realRegionCount != regions.size()) {
    errorMsg = "Regions moved during the snapshot '" +
                 ClientSnapshotDescriptionUtils.toString(snapshot) + "'. expected=" +
                 regions.size() + " snapshotted=" + realRegionCount + ".";
    LOG.error(errorMsg);
  }

  // Verify RegionInfo
  for (RegionInfo region : regions) {
    SnapshotRegionManifest regionManifest = regionManifests.get(region.getEncodedName());
    if (regionManifest == null) {
      // could happen due to a move or split race.
      String mesg = " No snapshot region directory found for region:" + region;
      if (errorMsg.isEmpty()) errorMsg = mesg;
      LOG.error(mesg);
      continue;
    }

    verifyRegionInfo(region, regionManifest);
  }

  if (!errorMsg.isEmpty()) {
    throw new CorruptedSnapshotException(errorMsg);
  }

  // Verify Snapshot HFiles
  // Requires the root directory file system as HFiles are stored in the root directory
  SnapshotReferenceUtil.verifySnapshot(services.getConfiguration(),
    CommonFSUtils.getRootDirFileSystem(services.getConfiguration()), manifest);
}
 
Example 18
Source Project: hbase   Source File: SnapshotManager.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Restore or Clone the specified snapshot
 * @param reqSnapshot
 * @param nonceKey unique identifier to prevent duplicated RPC
 * @throws IOException
 */
public long restoreOrCloneSnapshot(final SnapshotDescription reqSnapshot, final NonceKey nonceKey,
    final boolean restoreAcl) throws IOException {
  FileSystem fs = master.getMasterFileSystem().getFileSystem();
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(reqSnapshot, rootDir);

  // check if the snapshot exists
  if (!fs.exists(snapshotDir)) {
    LOG.error("A Snapshot named '" + reqSnapshot.getName() + "' does not exist.");
    throw new SnapshotDoesNotExistException(
      ProtobufUtil.createSnapshotDesc(reqSnapshot));
  }

  // Get snapshot info from file system. The reqSnapshot is a "fake" snapshotInfo with
  // just the snapshot "name" and table name to restore. It does not contains the "real" snapshot
  // information.
  SnapshotDescription snapshot = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest manifest = SnapshotManifest.open(master.getConfiguration(), fs,
      snapshotDir, snapshot);
  TableDescriptor snapshotTableDesc = manifest.getTableDescriptor();
  TableName tableName = TableName.valueOf(reqSnapshot.getTable());

  // sanity check the new table descriptor
  TableDescriptorChecker.sanityCheck(master.getConfiguration(), snapshotTableDesc);

  // stop tracking "abandoned" handlers
  cleanupSentinels();

  // Verify snapshot validity
  SnapshotReferenceUtil.verifySnapshot(master.getConfiguration(), fs, manifest);

  // Execute the restore/clone operation
  long procId;
  if (MetaTableAccessor.tableExists(master.getConnection(), tableName)) {
    procId = restoreSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey,
      restoreAcl);
  } else {
    procId =
        cloneSnapshot(reqSnapshot, tableName, snapshot, snapshotTableDesc, nonceKey, restoreAcl);
  }
  return procId;
}
 
Example 19
Source Project: hbase   Source File: DisabledTableSnapshotHandler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void snapshotRegions(List<Pair<RegionInfo, ServerName>> regionsAndLocations)
    throws IOException, KeeperException {
  try {
    // 1. get all the regions hosting this table.

    // extract each pair to separate lists
    Set<RegionInfo> regions = new HashSet<>();
    for (Pair<RegionInfo, ServerName> p : regionsAndLocations) {
      // Don't include non-default regions
      RegionInfo hri = p.getFirst();
      if (RegionReplicaUtil.isDefaultReplica(hri)) {
        regions.add(hri);
      }
    }
    // handle the mob files if any.
    boolean mobEnabled = MobUtils.hasMobColumns(htd);
    if (mobEnabled) {
      // snapshot the mob files as a offline region.
      RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(htd.getTableName());
      regions.add(mobRegionInfo);
    }

    // 2. for each region, write all the info to disk
    String msg = "Starting to write region info and WALs for regions for offline snapshot:"
        + ClientSnapshotDescriptionUtils.toString(snapshot);
    LOG.info(msg);
    status.setStatus(msg);

    ThreadPoolExecutor exec = SnapshotManifest.createExecutor(conf, "DisabledTableSnapshot");
    try {
      ModifyRegionUtils.editRegions(exec, regions, new ModifyRegionUtils.RegionEditTask() {
        @Override
        public void editRegion(final RegionInfo regionInfo) throws IOException {
          snapshotManifest.addRegion(CommonFSUtils.getTableDir(rootDir, snapshotTable),
            regionInfo);
        }
      });
    } finally {
      exec.shutdown();
    }
  } catch (Exception e) {
    // make sure we capture the exception to propagate back to the client later
    String reason = "Failed snapshot " + ClientSnapshotDescriptionUtils.toString(snapshot)
        + " due to exception:" + e.getMessage();
    ForeignException ee = new ForeignException(reason, e);
    monitor.receive(ee);
    status.abort("Snapshot of table: "+ snapshotTable + " failed because " + e.getMessage());
  } finally {
    LOG.debug("Marking snapshot" + ClientSnapshotDescriptionUtils.toString(snapshot)
        + " as finished.");
  }
}
 
Example 20
Source Project: hbase   Source File: TableSnapshotInputFormatImpl.java    License: Apache License 2.0 4 votes vote down vote up
public static List<InputSplit> getSplits(Configuration conf) throws IOException {
  String snapshotName = getSnapshotName(conf);

  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = rootDir.getFileSystem(conf);

  SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs);

  List<RegionInfo> regionInfos = getRegionInfosFromManifest(manifest);

  // TODO: mapred does not support scan as input API. Work around for now.
  Scan scan = extractScanFromConf(conf);
  // the temp dir where the snapshot is restored
  Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY));

  RegionSplitter.SplitAlgorithm splitAlgo = getSplitAlgo(conf);

  int numSplits = conf.getInt(NUM_SPLITS_PER_REGION, 1);

  return getSplits(scan, manifest, regionInfos, restoreDir, conf, splitAlgo, numSplits);
}
 
Example 21
Source Project: hbase   Source File: TableSnapshotInputFormatImpl.java    License: Apache License 2.0 4 votes vote down vote up
public static SnapshotManifest getSnapshotManifest(Configuration conf, String snapshotName,
    Path rootDir, FileSystem fs) throws IOException {
  Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir);
  SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  return SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
}
 
Example 22
Source Project: hbase   Source File: TableSnapshotInputFormatImpl.java    License: Apache License 2.0 4 votes vote down vote up
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest,
    List<RegionInfo> regionManifests, Path restoreDir, Configuration conf) throws IOException {
  return getSplits(scan, manifest, regionManifests, restoreDir, conf, null, 1);
}