org.apache.hadoop.hdfs.server.namenode.INodeDirectory Java Examples

The following examples show how to use org.apache.hadoop.hdfs.server.namenode.INodeDirectory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSnapshotDeletion.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void checkQuotaUsageComputation(final Path dirPath,
    final long expectedNs, final long expectedDs) throws IOException {
  INodeDirectory dirNode = getDir(fsdir, dirPath);
  assertTrue(dirNode.isQuotaSet());
  QuotaCounts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
      q.getNameSpace());
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
      q.getStorageSpace());
  QuotaCounts counts = new QuotaCounts.Builder().build();
  dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
      counts.getNameSpace());
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
      counts.getStorageSpace());
}
 
Example #2
Source File: FSImageFormatPBSnapshot.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Load the snapshots section from fsimage. Also add snapshottable feature
 * to snapshottable directories.
 */
public void loadSnapshotSection(InputStream in) throws IOException {
  SnapshotManager sm = fsn.getSnapshotManager();
  SnapshotSection section = SnapshotSection.parseDelimitedFrom(in);
  int snum = section.getNumSnapshots();
  sm.setNumSnapshots(snum);
  sm.setSnapshotCounter(section.getSnapshotCounter());
  for (long sdirId : section.getSnapshottableDirList()) {
    INodeDirectory dir = fsDir.getInode(sdirId).asDirectory();
    if (!dir.isSnapshottable()) {
      dir.addSnapshottableFeature();
    } else {
      // dir is root, and admin set root to snapshottable before
      dir.setSnapshotQuota(DirectorySnapshottableFeature.SNAPSHOT_LIMIT);
    }
    sm.addSnapshottable(dir);
  }
  loadSnapshots(in, snum);
}
 
Example #3
Source File: SnapshotFSImageFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Load {@link DirectoryDiff} from fsimage.
 * @param parent The directory that the SnapshotDiff belongs to.
 * @param in The {@link DataInput} instance to read.
 * @param loader The {@link Loader} instance that this loading procedure is 
 *               using.
 * @return A {@link DirectoryDiff}.
 */
private static DirectoryDiff loadDirectoryDiff(INodeDirectory parent,
    DataInput in, FSImageFormat.Loader loader) throws IOException {
  // 1. Read the full path of the Snapshot root to identify the Snapshot
  final Snapshot snapshot = loader.getSnapshot(in);

  // 2. Load DirectoryDiff#childrenSize
  int childrenSize = in.readInt();
  
  // 3. Load DirectoryDiff#snapshotINode 
  INodeDirectoryAttributes snapshotINode = loadSnapshotINodeInDirectoryDiff(
      snapshot, in, loader);
  
  // 4. Load the created list in SnapshotDiff#Diff
  List<INode> createdList = loadCreatedList(parent, in);
  
  // 5. Load the deleted list in SnapshotDiff#Diff
  List<INode> deletedList = loadDeletedList(parent, createdList, in, loader);
  
  // 6. Compose the SnapshotDiff
  List<DirectoryDiff> diffs = parent.getDiffs().asList();
  DirectoryDiff sdiff = new DirectoryDiff(snapshot.getId(), snapshotINode,
      diffs.isEmpty() ? null : diffs.get(0), childrenSize, createdList,
      deletedList, snapshotINode == snapshot.getRoot());
  return sdiff;
}
 
Example #4
Source File: SnapshotFSImageFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Load the deleted list from the fsimage.
 * 
 * @param parent The directory that the deleted list belongs to.
 * @param createdList The created list associated with the deleted list in 
 *                    the same Diff.
 * @param in The {@link DataInput} to read.
 * @param loader The {@link Loader} instance.
 * @return The deleted list.
 */
private static List<INode> loadDeletedList(INodeDirectory parent,
    List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
    throws IOException {
  int deletedSize = in.readInt();
  List<INode> deletedList = new ArrayList<INode>(deletedSize);
  for (int i = 0; i < deletedSize; i++) {
    final INode deleted = loader.loadINodeWithLocalName(true, in, true);
    deletedList.add(deleted);
    // set parent: the parent field of an INode in the deleted list is not 
    // useful, but set the parent here to be consistent with the original 
    // fsdir tree.
    deleted.setParent(parent);
    if (deleted.isFile()) {
      loader.updateBlocksMap(deleted.asFile());
    }
  }
  return deletedList;
}
 
Example #5
Source File: FSImageFormatPBSnapshot.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Load the snapshots section from fsimage. Also add snapshottable feature
 * to snapshottable directories.
 */
public void loadSnapshotSection(InputStream in) throws IOException {
  SnapshotManager sm = fsn.getSnapshotManager();
  SnapshotSection section = SnapshotSection.parseDelimitedFrom(in);
  int snum = section.getNumSnapshots();
  sm.setNumSnapshots(snum);
  sm.setSnapshotCounter(section.getSnapshotCounter());
  for (long sdirId : section.getSnapshottableDirList()) {
    INodeDirectory dir = fsDir.getInode(sdirId).asDirectory();
    if (!dir.isSnapshottable()) {
      dir.addSnapshottableFeature();
    } else {
      // dir is root, and admin set root to snapshottable before
      dir.setSnapshotQuota(DirectorySnapshottableFeature.SNAPSHOT_LIMIT);
    }
    sm.addSnapshottable(dir);
  }
  loadSnapshots(in, snum);
}
 
Example #6
Source File: SnapshotFSImageFormat.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Load a node stored in the created list from fsimage.
 * @param createdNodeName The name of the created node.
 * @param parent The directory that the created list belongs to.
 * @return The created node.
 */
public static INode loadCreated(byte[] createdNodeName,
    INodeDirectory parent) throws IOException {
  // the INode in the created list should be a reference to another INode
  // in posterior SnapshotDiffs or one of the current children
  for (DirectoryDiff postDiff : parent.getDiffs()) {
    final INode d = postDiff.getChildrenDiff().search(ListType.DELETED,
        createdNodeName);
    if (d != null) {
      return d;
    } // else go to the next SnapshotDiff
  } 
  // use the current child
  INode currentChild = parent.getChild(createdNodeName,
      Snapshot.CURRENT_STATE_ID);
  if (currentChild == null) {
    throw new IOException("Cannot find an INode associated with the INode "
        + DFSUtil.bytes2String(createdNodeName)
        + " in created list while loading FSImage.");
  }
  return currentChild;
}
 
Example #7
Source File: Helper.java    From NNAnalytics with Apache License 2.0 6 votes vote down vote up
/**
 * Returns function that maps an inode to its parent directory down to a specific depth.
 *
 * @param dirDepth the depth of the parent to fetch
 * @return a function
 */
public static Function<INode, String> getDirectoryAtDepthFunction(int dirDepth) {
  return node -> {
    try {
      INodeDirectory parent = node.getParent();
      int topParentDepth = new Path(parent.getFullPathName()).depth();
      if (topParentDepth < dirDepth) {
        return "NO_MAPPING";
      }
      for (int parentTravs = topParentDepth; parentTravs > dirDepth; parentTravs--) {
        parent = parent.getParent();
      }
      return parent.getFullPathName().intern();
    } catch (Exception e) {
      return "NO_MAPPING";
    }
  };
}
 
Example #8
Source File: SnapshotManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Compute the difference between two snapshots of a directory, or between a
 * snapshot of the directory and its current tree.
 */
public SnapshotDiffReport diff(final INodesInPath iip,
    final String snapshotRootPath, final String from,
    final String to) throws IOException {
  // Find the source root directory path where the snapshots were taken.
  // All the check for path has been included in the valueOf method.
  final INodeDirectory snapshotRoot = getSnapshottableRoot(iip);

  if ((from == null || from.isEmpty())
      && (to == null || to.isEmpty())) {
    // both fromSnapshot and toSnapshot indicate the current tree
    return new SnapshotDiffReport(snapshotRootPath, from, to,
        Collections.<DiffReportEntry> emptyList());
  }
  final SnapshotDiffInfo diffs = snapshotRoot
      .getDirectorySnapshottableFeature().computeDiff(snapshotRoot, from, to);
  return diffs != null ? diffs.generateReport() : new SnapshotDiffReport(
      snapshotRootPath, from, to, Collections.<DiffReportEntry> emptyList());
}
 
Example #9
Source File: TestSnapshotRename.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Check the correctness of snapshot list within snapshottable dir
 */
private void checkSnapshotList(INodeDirectory srcRoot,
    String[] sortedNames, String[] names) {
  assertTrue(srcRoot.isSnapshottable());
  ReadOnlyList<Snapshot> listByName = srcRoot
      .getDirectorySnapshottableFeature().getSnapshotList();
  assertEquals(sortedNames.length, listByName.size());
  for (int i = 0; i < listByName.size(); i++) {
    assertEquals(sortedNames[i], listByName.get(i).getRoot().getLocalName());
  }
  List<DirectoryDiff> listByTime = srcRoot.getDiffs().asList();
  assertEquals(names.length, listByTime.size());
  for (int i = 0; i < listByTime.size(); i++) {
    Snapshot s = srcRoot.getDirectorySnapshottableFeature().getSnapshotById(
        listByTime.get(i).getSnapshotId());
    assertEquals(names[i], s.getRoot().getLocalName());
  }
}
 
Example #10
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
QuotaCounts combinePosteriorAndCollectBlocks(
    final BlockStoragePolicySuite bsps,
    final INodeDirectory currentDir, final DirectoryDiff posterior,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  final QuotaCounts counts = new QuotaCounts.Builder().build();
  diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
    /** Collect blocks for deleted files. */
    @Override
    public void process(INode inode) {
      if (inode != null) {
        inode.computeQuotaUsage(bsps, counts, false);
        inode.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
      }
    }
  });
  return counts;
}
 
Example #11
Source File: Snapshot.java    From hadoop with Apache License 2.0 6 votes vote down vote up
Root(INodeDirectory other) {
  // Always preserve ACL, XAttr.
  super(other, false, Lists.newArrayList(
    Iterables.filter(Arrays.asList(other.getFeatures()), new Predicate<Feature>() {

      @Override
      public boolean apply(Feature input) {
        if (AclFeature.class.isInstance(input) 
            || XAttrFeature.class.isInstance(input)) {
          return true;
        }
        return false;
      }
      
    }))
    .toArray(new Feature[0]));
}
 
Example #12
Source File: TestSnapshotRename.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Rename snapshot(s), and check the correctness of the snapshot list within
 * {@link INodeDirectorySnapshottable}
 */
@Test (timeout=60000)
public void testSnapshotList() throws Exception {
  DFSTestUtil.createFile(hdfs, file1, BLOCKSIZE, REPLICATION, seed);
  // Create three snapshots for sub1
  SnapshotTestHelper.createSnapshot(hdfs, sub1, "s1");
  SnapshotTestHelper.createSnapshot(hdfs, sub1, "s2");
  SnapshotTestHelper.createSnapshot(hdfs, sub1, "s3");
  
  // Rename s3 to s22
  hdfs.renameSnapshot(sub1, "s3", "s22");
  // Check the snapshots list
  INodeDirectory srcRoot = fsdir.getINode(sub1.toString()).asDirectory();
  checkSnapshotList(srcRoot, new String[] { "s1", "s2", "s22" },
      new String[] { "s1", "s2", "s22" });
  
  // Rename s1 to s4
  hdfs.renameSnapshot(sub1, "s1", "s4");
  checkSnapshotList(srcRoot, new String[] { "s2", "s22", "s4" },
      new String[] { "s4", "s2", "s22" });
  
  // Rename s22 to s0
  hdfs.renameSnapshot(sub1, "s22", "s0");
  checkSnapshotList(srcRoot, new String[] { "s0", "s2", "s4" },
      new String[] { "s4", "s2", "s0" });
}
 
Example #13
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/** clear the created list */
private QuotaCounts destroyCreatedList(
    final BlockStoragePolicySuite bsps,
    final INodeDirectory currentINode,
    final BlocksMapUpdateInfo collectedBlocks,
    final List<INode> removedINodes) {
  QuotaCounts counts = new QuotaCounts.Builder().build();
  final List<INode> createdList = getList(ListType.CREATED);
  for (INode c : createdList) {
    c.computeQuotaUsage(bsps, counts, true);
    c.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
    // c should be contained in the children list, remove it
    currentINode.removeChild(c);
  }
  createdList.clear();
  return counts;
}
 
Example #14
Source File: TestSnapshotDeletion.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void checkQuotaUsageComputation(final Path dirPath,
    final long expectedNs, final long expectedDs) throws IOException {
  INodeDirectory dirNode = getDir(fsdir, dirPath);
  assertTrue(dirNode.isQuotaSet());
  QuotaCounts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
      q.getNameSpace());
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
      q.getStorageSpace());
  QuotaCounts counts = new QuotaCounts.Builder().build();
  dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
      counts.getNameSpace());
  assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
      counts.getStorageSpace());
}
 
Example #15
Source File: SnapshotFSImageFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Load the deleted list from the fsimage.
 * 
 * @param parent The directory that the deleted list belongs to.
 * @param createdList The created list associated with the deleted list in 
 *                    the same Diff.
 * @param in The {@link DataInput} to read.
 * @param loader The {@link Loader} instance.
 * @return The deleted list.
 */
private static List<INode> loadDeletedList(INodeDirectory parent,
    List<INode> createdList, DataInput in, FSImageFormat.Loader loader)
    throws IOException {
  int deletedSize = in.readInt();
  List<INode> deletedList = new ArrayList<INode>(deletedSize);
  for (int i = 0; i < deletedSize; i++) {
    final INode deleted = loader.loadINodeWithLocalName(true, in, true);
    deletedList.add(deleted);
    // set parent: the parent field of an INode in the deleted list is not 
    // useful, but set the parent here to be consistent with the original 
    // fsdir tree.
    deleted.setParent(parent);
    if (deleted.isFile()) {
      loader.updateBlocksMap(deleted.asFile());
    }
  }
  return deletedList;
}
 
Example #16
Source File: SnapshotFSImageFormat.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Load snapshots and snapshotQuota for a Snapshottable directory.
 *
 * @param snapshottableParent
 *          The snapshottable directory for loading.
 * @param numSnapshots
 *          The number of snapshots that the directory has.
 * @param loader
 *          The loader
 */
public static void loadSnapshotList(INodeDirectory snapshottableParent,
    int numSnapshots, DataInput in, FSImageFormat.Loader loader)
    throws IOException {
  DirectorySnapshottableFeature sf = snapshottableParent
      .getDirectorySnapshottableFeature();
  Preconditions.checkArgument(sf != null);
  for (int i = 0; i < numSnapshots; i++) {
    // read snapshots
    final Snapshot s = loader.getSnapshot(in);
    s.getRoot().setParent(snapshottableParent);
    sf.addSnapshot(s);
  }
  int snapshotQuota = in.readInt();
  snapshottableParent.setSnapshotQuota(snapshotQuota);
}
 
Example #17
Source File: SnapshotManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void checkNestedSnapshottable(INodeDirectory dir, String path)
    throws SnapshotException {
  if (allowNestedSnapshots) {
    return;
  }

  for(INodeDirectory s : snapshottables.values()) {
    if (s.isAncestorDirectory(dir)) {
      throw new SnapshotException(
          "Nested snapshottable directories not allowed: path=" + path
          + ", the subdirectory " + s.getFullPathName()
          + " is already a snapshottable directory.");
    }
    if (dir.isAncestorDirectory(s)) {
      throw new SnapshotException(
          "Nested snapshottable directories not allowed: path=" + path
          + ", the ancestor " + s.getFullPathName()
          + " is already a snapshottable directory.");
    }
  }
}
 
Example #18
Source File: SnapshotManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override // SnapshotStatsMXBean
public SnapshotInfo.Bean[] getSnapshots() {
  List<SnapshotInfo.Bean> beans = new ArrayList<SnapshotInfo.Bean>();
  for (INodeDirectory d : getSnapshottableDirs()) {
    for (Snapshot s : d.getDirectorySnapshottableFeature().getSnapshotList()) {
      beans.add(toBean(s));
    }
  }
  return beans.toArray(new SnapshotInfo.Bean[beans.size()]);
}
 
Example #19
Source File: FSImageFormatPBSnapshot.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void serializeDirDiffList(INodeDirectory dir,
    final List<INodeReference> refList, OutputStream out)
    throws IOException {
  DirectoryWithSnapshotFeature sf = dir.getDirectoryWithSnapshotFeature();
  if (sf != null) {
    List<DirectoryDiff> diffList = sf.getDiffs().asList();
    SnapshotDiffSection.DiffEntry entry = SnapshotDiffSection.DiffEntry
        .newBuilder().setInodeId(dir.getId()).setType(Type.DIRECTORYDIFF)
        .setNumOfDiff(diffList.size()).build();
    entry.writeDelimitedTo(out);
    for (int i = diffList.size() - 1; i >= 0; i--) { // reverse order!
      DirectoryDiff diff = diffList.get(i);
      SnapshotDiffSection.DirectoryDiff.Builder db = SnapshotDiffSection.
          DirectoryDiff.newBuilder().setSnapshotId(diff.getSnapshotId())
                       .setChildrenSize(diff.getChildrenSize())
                       .setIsSnapshotRoot(diff.isSnapshotRoot());
      INodeDirectoryAttributes copy = diff.snapshotINode;
      if (!diff.isSnapshotRoot() && copy != null) {
        db.setName(ByteString.copyFrom(copy.getLocalNameBytes()))
            .setSnapshotCopy(
                buildINodeDirectory(copy, parent.getSaverContext()));
      }
      // process created list and deleted list
      List<INode> created = diff.getChildrenDiff()
          .getList(ListType.CREATED);
      db.setCreatedListSize(created.size());
      List<INode> deleted = diff.getChildrenDiff().getList(ListType.DELETED);
      for (INode d : deleted) {
        if (d.isReference()) {
          refList.add(d.asReference());
          db.addDeletedINodeRef(refList.size() - 1);
        } else {
          db.addDeletedINode(d.getId());
        }
      }
      db.build().writeDelimitedTo(out);
      saveCreatedList(created, out);
    }
  }
}
 
Example #20
Source File: DirectorySnapshottableFeature.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Find the snapshot matching the given name.
 *
 * @param snapshotRoot The directory where snapshots were taken.
 * @param snapshotName The name of the snapshot.
 * @return The corresponding snapshot. Null if snapshotName is null or empty.
 * @throws SnapshotException If snapshotName is not null or empty, but there
 *           is no snapshot matching the name.
 */
private Snapshot getSnapshotByName(INodeDirectory snapshotRoot,
    String snapshotName) throws SnapshotException {
  Snapshot s = null;
  if (snapshotName != null && !snapshotName.isEmpty()) {
    final int index = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
    if (index < 0) {
      throw new SnapshotException("Cannot find the snapshot of directory "
          + snapshotRoot.getFullPathName() + " with name " + snapshotName);
    }
    s = snapshotsByNames.get(index);
  }
  return s;
}
 
Example #21
Source File: DirectoryWithSnapshotFeature.java    From big-c with Apache License 2.0 5 votes vote down vote up
public void clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode,
    final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
  // destroy its diff list
  for (DirectoryDiff diff : diffs) {
    diff.destroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks,
      removedINodes);
  }
  diffs.clear();
}
 
Example #22
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Add an inode into parent's children list. The caller of this method needs
 * to make sure that parent is in the given snapshot "latest".
 */
public boolean addChild(INodeDirectory parent, INode inode,
    boolean setModTime, int latestSnapshotId) throws QuotaExceededException {
  ChildrenDiff diff = diffs.checkAndAddLatestSnapshotDiff(latestSnapshotId,
      parent).diff;
  int undoInfo = diff.create(inode);

  final boolean added = parent.addChild(inode, setModTime,
      Snapshot.CURRENT_STATE_ID);
  if (!added) {
    diff.undoCreate(inode, undoInfo);
  }
  return added;
}
 
Example #23
Source File: DirectoryWithSnapshotFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * @return The children list of a directory in a snapshot.
 *         Since the snapshot is read-only, the logical view of the list is
 *         never changed although the internal data structure may mutate.
 */
private ReadOnlyList<INode> getChildrenList(final INodeDirectory currentDir) {
  return new ReadOnlyList<INode>() {
    private List<INode> children = null;

    private List<INode> initChildren() {
      if (children == null) {
        final ChildrenDiff combined = new ChildrenDiff();
        for (DirectoryDiff d = DirectoryDiff.this; d != null; 
            d = d.getPosterior()) {
          combined.combinePosterior(d.diff, null);
        }
        children = combined.apply2Current(ReadOnlyList.Util.asList(
            currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
      }
      return children;
    }

    @Override
    public Iterator<INode> iterator() {
      return initChildren().iterator();
    }

    @Override
    public boolean isEmpty() {
      return childrenSize == 0;
    }

    @Override
    public int size() {
      return childrenSize;
    }

    @Override
    public INode get(int i) {
      return initChildren().get(i);
    }
  };
}
 
Example #24
Source File: DirectorySnapshottableFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Compute the difference between two snapshots (or a snapshot and the current
 * directory) of the directory.
 *
 * @param from The name of the start point of the comparison. Null indicating
 *          the current tree.
 * @param to The name of the end point. Null indicating the current tree.
 * @return The difference between the start/end points.
 * @throws SnapshotException If there is no snapshot matching the starting
 *           point, or if endSnapshotName is not null but cannot be identified
 *           as a previous snapshot.
 */
SnapshotDiffInfo computeDiff(final INodeDirectory snapshotRoot,
    final String from, final String to) throws SnapshotException {
  Snapshot fromSnapshot = getSnapshotByName(snapshotRoot, from);
  Snapshot toSnapshot = getSnapshotByName(snapshotRoot, to);
  // if the start point is equal to the end point, return null
  if (from.equals(to)) {
    return null;
  }
  SnapshotDiffInfo diffs = new SnapshotDiffInfo(snapshotRoot, fromSnapshot,
      toSnapshot);
  computeDiffRecursively(snapshotRoot, snapshotRoot, new ArrayList<byte[]>(),
      diffs);
  return diffs;
}
 
Example #25
Source File: TestNestedSnapshots.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Test {@link Snapshot#ID_COMPARATOR}.
 */
@Test (timeout=300000)
public void testIdCmp() {
  final PermissionStatus perm = PermissionStatus.createImmutable(
      "user", "group", FsPermission.createImmutable((short)0));
  final INodeDirectory snapshottable = new INodeDirectory(0,
      DFSUtil.string2Bytes("foo"), perm, 0L);
  snapshottable.addSnapshottableFeature();
  final Snapshot[] snapshots = {
    new Snapshot(1, "s1", snapshottable),
    new Snapshot(1, "s1", snapshottable),
    new Snapshot(2, "s2", snapshottable),
    new Snapshot(2, "s2", snapshottable),
  };

  Assert.assertEquals(0, Snapshot.ID_COMPARATOR.compare(null, null));
  for(Snapshot s : snapshots) {
    Assert.assertTrue(Snapshot.ID_COMPARATOR.compare(null, s) > 0);
    Assert.assertTrue(Snapshot.ID_COMPARATOR.compare(s, null) < 0);
    
    for(Snapshot t : snapshots) {
      final int expected = s.getRoot().getLocalName().compareTo(
          t.getRoot().getLocalName());
      final int computed = Snapshot.ID_COMPARATOR.compare(s, t);
      Assert.assertEquals(expected > 0, computed > 0);
      Assert.assertEquals(expected == 0, computed == 0);
      Assert.assertEquals(expected < 0, computed < 0);
    }
  }
}
 
Example #26
Source File: FSImageFormatPBSnapshot.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Load the created list in a DirectoryDiff */
private List<INode> loadCreatedList(InputStream in, INodeDirectory dir,
    int size) throws IOException {
  List<INode> clist = new ArrayList<INode>(size);
  for (long c = 0; c < size; c++) {
    CreatedListEntry entry = CreatedListEntry.parseDelimitedFrom(in);
    INode created = SnapshotFSImageFormat.loadCreated(entry.getName()
        .toByteArray(), dir);
    clist.add(created);
  }
  return clist;
}
 
Example #27
Source File: DirectorySnapshottableFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public ContentSummaryComputationContext computeContentSummary(
    final BlockStoragePolicySuite bsps,
    final INodeDirectory snapshotRoot,
    final ContentSummaryComputationContext summary) {
  snapshotRoot.computeContentSummary(summary);
  summary.getCounts().addContent(Content.SNAPSHOT, snapshotsByNames.size());
  summary.getCounts().addContent(Content.SNAPSHOTTABLE_DIRECTORY, 1);
  return summary;
}
 
Example #28
Source File: TestRenameWithSnapshots.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Rename a single file across snapshottable dirs.
 */
@Test (timeout=60000)
public void testRenameFileAcrossSnapshottableDirs() throws Exception {
  final Path sdir1 = new Path("/dir1");
  final Path sdir2 = new Path("/dir2");
  hdfs.mkdirs(sdir1);
  hdfs.mkdirs(sdir2);
  final Path foo = new Path(sdir2, "foo");
  DFSTestUtil.createFile(hdfs, foo, BLOCKSIZE, REPL, SEED);
  
  SnapshotTestHelper.createSnapshot(hdfs, sdir1, "s1");
  SnapshotTestHelper.createSnapshot(hdfs, sdir2, "s2");
  hdfs.createSnapshot(sdir1, "s3");
  
  final Path newfoo = new Path(sdir1, "foo");
  hdfs.rename(foo, newfoo);
  
  // change the replication factor of foo
  hdfs.setReplication(newfoo, REPL_1);
  
  // /dir2/.snapshot/s2/foo should still work
  final Path foo_s2 = SnapshotTestHelper.getSnapshotPath(sdir2, "s2",
      "foo");
  assertTrue(hdfs.exists(foo_s2));
  FileStatus status = hdfs.getFileStatus(foo_s2);
  assertEquals(REPL, status.getReplication());
  
  final Path foo_s3 = SnapshotTestHelper.getSnapshotPath(sdir1, "s3",
      "foo");
  assertFalse(hdfs.exists(foo_s3));
  INodeDirectory sdir2Node = fsdir.getINode(sdir2.toString()).asDirectory();
  Snapshot s2 = sdir2Node.getSnapshot(DFSUtil.string2Bytes("s2"));
  INodeFile sfoo = fsdir.getINode(newfoo.toString()).asFile();
  assertEquals(s2.getId(), sfoo.getDiffs().getLastSnapshotId());
}
 
Example #29
Source File: DirectorySnapshottableFeature.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/** Add a snapshot. */
public Snapshot addSnapshot(INodeDirectory snapshotRoot, int id, String name)
    throws SnapshotException, QuotaExceededException {
  //check snapshot quota
  final int n = getNumSnapshots();
  if (n + 1 > snapshotQuota) {
    throw new SnapshotException("Failed to add snapshot: there are already "
        + n + " snapshot(s) and the snapshot quota is "
        + snapshotQuota);
  }
  final Snapshot s = new Snapshot(id, name, snapshotRoot);
  final byte[] nameBytes = s.getRoot().getLocalNameBytes();
  final int i = searchSnapshot(nameBytes);
  if (i >= 0) {
    throw new SnapshotException("Failed to add snapshot: there is already a "
        + "snapshot with the same name \"" + Snapshot.getSnapshotName(s) + "\".");
  }

  final DirectoryDiff d = getDiffs().addDiff(id, snapshotRoot);
  d.setSnapshotRoot(s.getRoot());
  snapshotsByNames.add(-i - 1, s);

  // set modification time
  final long now = Time.now();
  snapshotRoot.updateModificationTime(now, Snapshot.CURRENT_STATE_ID);
  s.getRoot().setModificationTime(now, Snapshot.CURRENT_STATE_ID);
  return s;
}
 
Example #30
Source File: DirectoryWithSnapshotFeature.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * @return The children list of a directory in a snapshot.
 *         Since the snapshot is read-only, the logical view of the list is
 *         never changed although the internal data structure may mutate.
 */
private ReadOnlyList<INode> getChildrenList(final INodeDirectory currentDir) {
  return new ReadOnlyList<INode>() {
    private List<INode> children = null;

    private List<INode> initChildren() {
      if (children == null) {
        final ChildrenDiff combined = new ChildrenDiff();
        for (DirectoryDiff d = DirectoryDiff.this; d != null; 
            d = d.getPosterior()) {
          combined.combinePosterior(d.diff, null);
        }
        children = combined.apply2Current(ReadOnlyList.Util.asList(
            currentDir.getChildrenList(Snapshot.CURRENT_STATE_ID)));
      }
      return children;
    }

    @Override
    public Iterator<INode> iterator() {
      return initChildren().iterator();
    }

    @Override
    public boolean isEmpty() {
      return childrenSize == 0;
    }

    @Override
    public int size() {
      return childrenSize;
    }

    @Override
    public INode get(int i) {
      return initChildren().get(i);
    }
  };
}