org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestHFileOutputFormat2.java    From hbase with Apache License 2.0 6 votes vote down vote up
private String getStoragePolicyNameForOldHDFSVersion(FileSystem fs, Path path) {
  try {
    if (fs instanceof DistributedFileSystem) {
      DistributedFileSystem dfs = (DistributedFileSystem) fs;
      HdfsFileStatus status = dfs.getClient().getFileInfo(path.toUri().getPath());
      if (null != status) {
        byte storagePolicyId = status.getStoragePolicy();
        Field idUnspecified = BlockStoragePolicySuite.class.getField("ID_UNSPECIFIED");
        if (storagePolicyId != idUnspecified.getByte(BlockStoragePolicySuite.class)) {
          BlockStoragePolicy[] policies = dfs.getStoragePolicies();
          for (BlockStoragePolicy policy : policies) {
            if (policy.getId() == storagePolicyId) {
              return policy.getName();
            }
          }
        }
      }
    }
  } catch (Throwable e) {
    LOG.warn("failed to get block storage policy of [" + path + "]", e);
  }

  return null;
}
 
Example #2
Source File: FSDirRenameOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
 * dstInodes[dstInodes.length-1]
 */
private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src,
    INodesInPath dst) throws QuotaExceededException {
  if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
    // Do not check quota if edits log is still being processed
    return;
  }
  int i = 0;
  while(src.getINode(i) == dst.getINode(i)) { i++; }
  // src[i - 1] is the last common ancestor.
  BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
  final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);

  // Reduce the required quota by dst that is being removed
  final INode dstINode = dst.getLastINode();
  if (dstINode != null) {
    delta.subtract(dstINode.computeQuotaUsage(bsps));
  }
  FSDirectory.verifyQuota(dst, dst.length() - 1, delta, src.getINode(i - 1));
}
 
Example #3
Source File: INodeDirectory.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void destroyAndCollectBlocks(final BlockStoragePolicySuite bsps,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
  if (sf != null) {
    sf.clear(bsps, this, collectedBlocks, removedINodes);
  }
  for (INode child : getChildrenList(Snapshot.CURRENT_STATE_ID)) {
    child.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
  }
  if (getAclFeature() != null) {
    AclStorage.removeAclFeature(getAclFeature());
  }
  clear();
  removedINodes.add(this);
}
 
Example #4
Source File: Mover.java    From hadoop with Apache License 2.0 6 votes vote down vote up
Mover(NameNodeConnector nnc, Configuration conf, AtomicInteger retryCount) {
  final long movedWinWidth = conf.getLong(
      DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY,
      DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_DEFAULT);
  final int moverThreads = conf.getInt(
      DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY,
      DFSConfigKeys.DFS_MOVER_MOVERTHREADS_DEFAULT);
  final int maxConcurrentMovesPerNode = conf.getInt(
      DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
      DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT);
  this.retryMaxAttempts = conf.getInt(
      DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY,
      DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT);
  this.retryCount = retryCount;
  this.dispatcher = new Dispatcher(nnc, Collections.<String> emptySet(),
      Collections.<String> emptySet(), movedWinWidth, moverThreads, 0,
      maxConcurrentMovesPerNode, conf);
  this.storages = new StorageMap();
  this.targetPaths = nnc.getTargetPaths();
  this.blockStoragePolicies = new BlockStoragePolicy[1 <<
      BlockStoragePolicySuite.ID_BIT_LENGTH];
}
 
Example #5
Source File: TestStoragePolicySummary.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleHots() {
  BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
  StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
  BlockStoragePolicy hot = bsps.getPolicy("HOT");
  sts.add(new StorageType[]{StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,
      StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,
      StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
  Map<String, Long> actualOutput = convertToStringMap(sts);
  Assert.assertEquals(4,actualOutput.size());
  Map<String, Long>  expectedOutput = new HashMap<>();
  expectedOutput.put("HOT|DISK:1(HOT)", 1l);
  expectedOutput.put("HOT|DISK:2(HOT)", 1l);
  expectedOutput.put("HOT|DISK:3(HOT)", 1l);
  expectedOutput.put("HOT|DISK:4(HOT)", 1l);
  Assert.assertEquals(expectedOutput,actualOutput);
}
 
Example #6
Source File: FSDirRenameOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
boolean cleanDst(BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks)
    throws QuotaExceededException {
  Preconditions.checkState(oldDstChild != null);
  List<INode> removedINodes = new ChunkedArrayList<>();
  final boolean filesDeleted;
  if (!oldDstChild.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) {
    oldDstChild.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
    filesDeleted = true;
  } else {
    filesDeleted = oldDstChild.cleanSubtree(bsps, Snapshot.CURRENT_STATE_ID,
        dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes)
        .getNameSpace() >= 0;
  }
  fsd.getFSNamesystem().removeLeasesAndINodes(src, removedINodes, false);
  return filesDeleted;
}
 
Example #7
Source File: FSDirRenameOp.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Verify quota for rename operation where srcInodes[srcInodes.length-1] moves
 * dstInodes[dstInodes.length-1]
 */
private static void verifyQuotaForRename(FSDirectory fsd, INodesInPath src,
    INodesInPath dst) throws QuotaExceededException {
  if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
    // Do not check quota if edits log is still being processed
    return;
  }
  int i = 0;
  while(src.getINode(i) == dst.getINode(i)) { i++; }
  // src[i - 1] is the last common ancestor.
  BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
  final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);

  // Reduce the required quota by dst that is being removed
  final INode dstINode = dst.getLastINode();
  if (dstINode != null) {
    delta.subtract(dstINode.computeQuotaUsage(bsps));
  }
  FSDirectory.verifyQuota(dst, dst.length() - 1, delta, src.getINode(i - 1));
}
 
Example #8
Source File: Mover.java    From big-c with Apache License 2.0 6 votes vote down vote up
Mover(NameNodeConnector nnc, Configuration conf, AtomicInteger retryCount) {
  final long movedWinWidth = conf.getLong(
      DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_KEY,
      DFSConfigKeys.DFS_MOVER_MOVEDWINWIDTH_DEFAULT);
  final int moverThreads = conf.getInt(
      DFSConfigKeys.DFS_MOVER_MOVERTHREADS_KEY,
      DFSConfigKeys.DFS_MOVER_MOVERTHREADS_DEFAULT);
  final int maxConcurrentMovesPerNode = conf.getInt(
      DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_KEY,
      DFSConfigKeys.DFS_DATANODE_BALANCE_MAX_NUM_CONCURRENT_MOVES_DEFAULT);
  this.retryMaxAttempts = conf.getInt(
      DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_KEY,
      DFSConfigKeys.DFS_MOVER_RETRY_MAX_ATTEMPTS_DEFAULT);
  this.retryCount = retryCount;
  this.dispatcher = new Dispatcher(nnc, Collections.<String> emptySet(),
      Collections.<String> emptySet(), movedWinWidth, moverThreads, 0,
      maxConcurrentMovesPerNode, conf);
  this.storages = new StorageMap();
  this.targetPaths = nnc.getTargetPaths();
  this.blockStoragePolicies = new BlockStoragePolicy[1 <<
      BlockStoragePolicySuite.ID_BIT_LENGTH];
}
 
Example #9
Source File: FSDirStatAndListingOp.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static HdfsFileStatus getFileInfo(
    FSDirectory fsd, String src, boolean resolveLink, boolean isRawPath,
    boolean includeStoragePolicy)
  throws IOException {
  String srcs = FSDirectory.normalizePath(src);
  if (srcs.endsWith(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR)) {
    if (fsd.getINode4DotSnapshot(srcs) != null) {
      return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
          HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
          BlockStoragePolicySuite.ID_UNSPECIFIED);
    }
    return null;
  }

  fsd.readLock();
  try {
    final INodesInPath iip = fsd.getINodesInPath(srcs, resolveLink);
    return getFileInfo(fsd, src, iip, isRawPath, includeStoragePolicy);
  } finally {
    fsd.readUnlock();
  }
}
 
Example #10
Source File: DirectoryWithSnapshotFeature.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
QuotaCounts combinePosteriorAndCollectBlocks(
    final BlockStoragePolicySuite bsps,
    final INodeDirectory currentDir, final DirectoryDiff posterior,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  final QuotaCounts counts = new QuotaCounts.Builder().build();
  diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
    /** Collect blocks for deleted files. */
    @Override
    public void process(INode inode) {
      if (inode != null) {
        inode.computeQuotaUsage(bsps, counts, false);
        inode.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
      }
    }
  });
  return counts;
}
 
Example #11
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/** clear the created list */
private QuotaCounts destroyCreatedList(
    final BlockStoragePolicySuite bsps,
    final INodeDirectory currentINode,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  QuotaCounts counts = new QuotaCounts.Builder().build();
  final List<INode> createdList = getList(ListType.CREATED);
  for (INode c : createdList) {
    c.computeQuotaUsage(bsps, counts, true);
    c.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
    // c should be contained in the children list, remove it
    currentINode.removeChild(c);
  }
  createdList.clear();
  return counts;
}
 
Example #12
Source File: TestStoragePolicySummary.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleHotsWithDifferentCounts() {
  BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
  StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
  BlockStoragePolicy hot = bsps.getPolicy("HOT");
  sts.add(new StorageType[]{StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,
      StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,
      StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,
      StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
  Map<String, Long> actualOutput = convertToStringMap(sts);
  Assert.assertEquals(4,actualOutput.size());
  Map<String, Long> expectedOutput = new HashMap<>();
  expectedOutput.put("HOT|DISK:1(HOT)", 1l);
  expectedOutput.put("HOT|DISK:2(HOT)", 2l);
  expectedOutput.put("HOT|DISK:3(HOT)", 2l);
  expectedOutput.put("HOT|DISK:4(HOT)", 1l);
  Assert.assertEquals(expectedOutput,actualOutput);
}
 
Example #13
Source File: INodeReference.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps,
    byte blockStoragePolicyId, QuotaCounts counts, boolean useCache,
    int lastSnapshotId) {
  // if this.lastSnapshotId < lastSnapshotId, the rename of the referred 
  // node happened before the rename of its ancestor. This should be 
  // impossible since for WithName node we only count its children at the 
  // time of the rename. 
  Preconditions.checkState(lastSnapshotId == Snapshot.CURRENT_STATE_ID
      || this.lastSnapshotId >= lastSnapshotId);
  final INode referred = this.getReferredINode().asReference()
      .getReferredINode();
  // We will continue the quota usage computation using the same snapshot id
  // as time line (if the given snapshot id is valid). Also, we cannot use 
  // cache for the referred node since its cached quota may have already 
  // been updated by changes in the current tree.
  int id = lastSnapshotId != Snapshot.CURRENT_STATE_ID ? 
      lastSnapshotId : this.lastSnapshotId;
  return referred.computeQuotaUsage(bsps, blockStoragePolicyId, counts,
      false, id);
}
 
Example #14
Source File: INodeDirectory.java    From hadoop with Apache License 2.0 6 votes vote down vote up
void setQuota(BlockStoragePolicySuite bsps, long nsQuota, long ssQuota, StorageType type) {
  DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
  if (quota != null) {
    // already has quota; so set the quota to the new values
    if (type != null) {
      quota.setQuota(ssQuota, type);
    } else {
      quota.setQuota(nsQuota, ssQuota);
    }
    if (!isQuotaSet() && !isRoot()) {
      removeFeature(quota);
    }
  } else {
    final QuotaCounts c = computeQuotaUsage(bsps);
    DirectoryWithQuotaFeature.Builder builder =
        new DirectoryWithQuotaFeature.Builder().nameSpaceQuota(nsQuota);
    if (type != null) {
      builder.typeQuota(type, ssQuota);
    } else {
      builder.storageSpaceQuota(ssQuota);
    }
    addDirectoryWithQuotaFeature(builder.build()).setSpaceConsumed(c);
  }
}
 
Example #15
Source File: DirectoryWithSnapshotFeature.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode,
    final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
  // destroy its diff list
  for (DirectoryDiff diff : diffs) {
    diff.destroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks,
      removedINodes);
  }
  diffs.clear();
}
 
Example #16
Source File: SnapshottableDirectoryStatus.java    From big-c with Apache License 2.0 5 votes vote down vote up
public SnapshottableDirectoryStatus(long modification_time, long access_time,
    FsPermission permission, String owner, String group, byte[] localName,
    long inodeId, int childrenNum,
    int snapshotNumber, int snapshotQuota, byte[] parentFullPath) {
  this.dirStatus = new HdfsFileStatus(0, true, 0, 0, modification_time,
      access_time, permission, owner, group, null, localName, inodeId,
      childrenNum, null, BlockStoragePolicySuite.ID_UNSPECIFIED);
  this.snapshotNumber = snapshotNumber;
  this.snapshotQuota = snapshotQuota;
  this.parentFullPath = parentFullPath;
}
 
Example #17
Source File: FSDirStatAndListingOp.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Currently we only support "ls /xxx/.snapshot" which will return all the
 * snapshots of a directory. The FSCommand Ls will first call getFileInfo to
 * make sure the file/directory exists (before the real getListing call).
 * Since we do not have a real INode for ".snapshot", we return an empty
 * non-null HdfsFileStatus here.
 */
private static HdfsFileStatus getFileInfo4DotSnapshot(
    FSDirectory fsd, String src)
    throws UnresolvedLinkException {
  if (fsd.getINode4DotSnapshot(src) != null) {
    return new HdfsFileStatus(0, true, 0, 0, 0, 0, null, null, null, null,
        HdfsFileStatus.EMPTY_NAME, -1L, 0, null,
        BlockStoragePolicySuite.ID_UNSPECIFIED);
  }
  return null;
}
 
Example #18
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode,
    final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
  // destroy its diff list
  for (DirectoryDiff diff : diffs) {
    diff.destroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks,
      removedINodes);
  }
  diffs.clear();
}
 
Example #19
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public QuotaCounts computeQuotaUsage4CurrentDirectory(
    BlockStoragePolicySuite bsps, byte storagePolicyId,
    QuotaCounts counts) {
  for(DirectoryDiff d : diffs) {
    for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
      final byte childPolicyId = deleted.getStoragePolicyIDForQuota(storagePolicyId);
      deleted.computeQuotaUsage(bsps, childPolicyId, counts, false,
          Snapshot.CURRENT_STATE_ID);
    }
  }
  return counts;
}
 
Example #20
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void computeContentSummary4Snapshot(final BlockStoragePolicySuite bsps,
    final ContentCounts counts) {
  // Create a new blank summary context for blocking processing of subtree.
  ContentSummaryComputationContext summary = 
      new ContentSummaryComputationContext(bsps);
  for(DirectoryDiff d : diffs) {
    for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
      deleted.computeContentSummary(summary);
    }
  }
  // Add the counts from deleted trees.
  counts.addContents(summary.getCounts());
  // Add the deleted directory count.
  counts.addContent(Content.DIRECTORY, diffs.asList().size());
}
 
Example #21
Source File: TestStoragePolicySummary.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortInDescendingOrder() {
  BlockStoragePolicySuite bsps = BlockStoragePolicySuite.createDefaultSuite();
  StoragePolicySummary sts = new StoragePolicySummary(bsps.getAllPolicies());
  BlockStoragePolicy hot = bsps.getPolicy("HOT");
  BlockStoragePolicy warm = bsps.getPolicy("WARM");
  BlockStoragePolicy cold = bsps.getPolicy("COLD");
  //DISK:3
  sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
  sts.add(new StorageType[]{StorageType.DISK,StorageType.DISK,StorageType.DISK},hot);
  //DISK:1,ARCHIVE:2
  sts.add(new StorageType[]{StorageType.DISK,
      StorageType.ARCHIVE,StorageType.ARCHIVE},warm);
  sts.add(new StorageType[]{StorageType.ARCHIVE,
      StorageType.DISK,StorageType.ARCHIVE},warm);
  sts.add(new StorageType[]{StorageType.ARCHIVE,
      StorageType.ARCHIVE,StorageType.DISK},warm);
  //ARCHIVE:3
  sts.add(new StorageType[]{StorageType.ARCHIVE,
      StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
  sts.add(new StorageType[]{StorageType.ARCHIVE,
      StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
  sts.add(new StorageType[]{StorageType.ARCHIVE,
      StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
  sts.add(new StorageType[]{StorageType.ARCHIVE,
      StorageType.ARCHIVE,StorageType.ARCHIVE},cold);
  Map<String, Long> actualOutput = convertToStringMap(sts);
  Assert.assertEquals(3,actualOutput.size());
  Map<String, Long>  expectedOutput = new LinkedHashMap<>();
  expectedOutput.put("COLD|ARCHIVE:3(COLD)", 4l);
  expectedOutput.put("WARM|DISK:1,ARCHIVE:2(WARM)", 3l);
  expectedOutput.put("HOT|DISK:3(HOT)", 2l);
  Assert.assertEquals(expectedOutput.toString(),actualOutput.toString());
}
 
Example #22
Source File: INode.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Count subtree {@link Quota#NAMESPACE} and {@link Quota#STORAGESPACE} usages.
 * Entry point for FSDirectory where blockStoragePolicyId is given its initial
 * value.
 */
public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps) {
  final byte storagePolicyId = isSymlink() ?
      BlockStoragePolicySuite.ID_UNSPECIFIED : getStoragePolicyID();
  return computeQuotaUsage(bsps, storagePolicyId,
      new QuotaCounts.Builder().build(), true, Snapshot.CURRENT_STATE_ID);
}
 
Example #23
Source File: DirectorySnapshottableFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Remove the snapshot with the given name from {@link #snapshotsByNames},
 * and delete all the corresponding DirectoryDiff.
 *
 * @param snapshotRoot The directory where we take snapshots
 * @param snapshotName The name of the snapshot to be removed
 * @param collectedBlocks Used to collect information to update blocksMap
 * @return The removed snapshot. Null if no snapshot with the given name
 *         exists.
 */
public Snapshot removeSnapshot(BlockStoragePolicySuite bsps, INodeDirectory snapshotRoot,
    String snapshotName, BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) throws SnapshotException {
  final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
  if (i < 0) {
    throw new SnapshotException("Cannot delete snapshot " + snapshotName
        + " from path " + snapshotRoot.getFullPathName()
        + ": the snapshot does not exist.");
  } else {
    final Snapshot snapshot = snapshotsByNames.get(i);
    int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId());
    try {
      QuotaCounts counts = snapshotRoot.cleanSubtree(bsps, snapshot.getId(),
          prior, collectedBlocks, removedINodes);
      INodeDirectory parent = snapshotRoot.getParent();
      if (parent != null) {
        // there will not be any WithName node corresponding to the deleted
        // snapshot, thus only update the quota usage in the current tree
        parent.addSpaceConsumed(counts.negation(), true);
      }
    } catch(QuotaExceededException e) {
      INode.LOG.error("BUG: removeSnapshot increases namespace usage.", e);
    }
    // remove from snapshotsByNames after successfully cleaning the subtree
    snapshotsByNames.remove(i);
    return snapshot;
  }
}
 
Example #24
Source File: FSDirRenameOp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
void updateQuotasInSourceTree(BlockStoragePolicySuite bsps) throws QuotaExceededException {
  // update the quota usage in src tree
  if (isSrcInSnapshot) {
    // get the counts after rename
    QuotaCounts newSrcCounts = srcChild.computeQuotaUsage(bsps,
        new QuotaCounts.Builder().build(), false);
    newSrcCounts.subtract(oldSrcCounts);
    srcParent.addSpaceConsumed(newSrcCounts, false);
  }
}
 
Example #25
Source File: FileDiff.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
QuotaCounts combinePosteriorAndCollectBlocks(
    BlockStoragePolicySuite bsps, INodeFile currentINode,
    FileDiff posterior, BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  FileWithSnapshotFeature sf = currentINode.getFileWithSnapshotFeature();
  assert sf != null : "FileWithSnapshotFeature is null";
  return sf.updateQuotaAndCollectBlocks(
      bsps, currentINode, posterior, collectedBlocks, removedINodes);
}
 
Example #26
Source File: INodeReference.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override // used by WithCount
public void destroyAndCollectBlocks(
    BlockStoragePolicySuite bsps,
    BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
  if (removeReference(this) <= 0) {
    referred.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
  }
}
 
Example #27
Source File: INodeSymlink.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public QuotaCounts computeQuotaUsage(
    BlockStoragePolicySuite bsps, byte blockStoragePolicyId,
    QuotaCounts counts, boolean useCache, int lastSnapshotId) {
  counts.addNameSpace(1);
  return counts;
}
 
Example #28
Source File: FSDirRenameOp.java    From big-c with Apache License 2.0 5 votes vote down vote up
void restoreDst(BlockStoragePolicySuite bsps) throws QuotaExceededException {
  Preconditions.checkState(oldDstChild != null);
  final INodeDirectory dstParent = dstParentIIP.getLastINode().asDirectory();
  if (dstParent.isWithSnapshot()) {
    dstParent.undoRename4DstParent(bsps, oldDstChild, dstIIP.getLatestSnapshotId());
  } else {
    fsd.addLastINodeNoQuotaCheck(dstParentIIP, oldDstChild);
  }
  if (oldDstChild != null && oldDstChild.isReference()) {
    final INodeReference removedDstRef = oldDstChild.asReference();
    final INodeReference.WithCount wc = (INodeReference.WithCount)
        removedDstRef.getReferredINode().asReference();
    wc.addReference(removedDstRef);
  }
}
 
Example #29
Source File: INodeReference.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps, int snapshot, int prior,
    BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
  if (snapshot == Snapshot.CURRENT_STATE_ID
      && prior == Snapshot.NO_SNAPSHOT_ID) {
    QuotaCounts counts = new QuotaCounts.Builder().build();
    this.computeQuotaUsage(bsps, counts, true);
    destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
    return counts;
  } else {
    // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to 
    // the previous WithName instance
    if (prior == Snapshot.NO_SNAPSHOT_ID) {
      prior = getPriorSnapshot(this);
    }
    // if prior is not NO_SNAPSHOT_ID, and prior is not before the
    // to-be-deleted snapshot, we can quit here and leave the snapshot
    // deletion work to the src tree of rename
    if (snapshot != Snapshot.CURRENT_STATE_ID
        && prior != Snapshot.NO_SNAPSHOT_ID
        && Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
      return new QuotaCounts.Builder().build();
    }
    return getReferredINode().cleanSubtree(bsps, snapshot, prior,
        collectedBlocks, removedINodes);
  }
}
 
Example #30
Source File: INodeReference.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
    final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  // since WithName node resides in deleted list acting as a snapshot copy,
  // the parameter snapshot must be non-null
  Preconditions.checkArgument(snapshot != Snapshot.CURRENT_STATE_ID);
  // if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to the
  // previous WithName instance
  if (prior == Snapshot.NO_SNAPSHOT_ID) {
    prior = getPriorSnapshot(this);
  }
  
  if (prior != Snapshot.NO_SNAPSHOT_ID
      && Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
    return new QuotaCounts.Builder().build();
  }

  QuotaCounts counts = getReferredINode().cleanSubtree(bsps, snapshot, prior,
      collectedBlocks, removedINodes);
  INodeReference ref = getReferredINode().getParentReference();
  if (ref != null) {
    try {
      ref.addSpaceConsumed(counts.negation(), true);
    } catch (QuotaExceededException e) {
      Log.warn("Should not have QuotaExceededException");
    }
  }
  
  if (snapshot < lastSnapshotId) {
    // for a WithName node, when we compute its quota usage, we only count
    // in all the nodes existing at the time of the corresponding rename op.
    // Thus if we are deleting a snapshot before/at the snapshot associated 
    // with lastSnapshotId, we do not need to update the quota upwards.
    counts = new QuotaCounts.Builder().build();
  }
  return counts;
}