Java Code Examples for org.apache.hadoop.tools.DistCpOptions.FileAttribute

The following examples show how to use org.apache.hadoop.tools.DistCpOptions.FileAttribute. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: RetriableFileCopyCommand.java    License: Apache License 2.0 6 votes vote down vote up
private long copyToFile(Path targetPath, FileSystem targetFS,
    FileStatus sourceFileStatus, long sourceOffset, Mapper.Context context,
    EnumSet<FileAttribute> fileAttributes, final FileChecksum sourceChecksum)
    throws IOException {
  FsPermission permission = FsPermission.getFileDefault().applyUMask(
      FsPermission.getUMask(targetFS.getConf()));
  final OutputStream outStream;
  if (action == FileAction.OVERWRITE) {
    final short repl = getReplicationFactor(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    final long blockSize = getBlockSize(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    FSDataOutputStream out = targetFS.create(targetPath, permission,
        EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
        BUFFER_SIZE, repl, blockSize, context,
        getChecksumOpt(fileAttributes, sourceChecksum));
    outStream = new BufferedOutputStream(out);
  } else {
    outStream = new BufferedOutputStream(targetFS.append(targetPath,
        BUFFER_SIZE));
  }
  return copyBytes(sourceFileStatus, sourceOffset, outStream, BUFFER_SIZE,
      context);
}
 
Example 2
Source Project: hadoop   Source File: CopyMapper.java    License: Apache License 2.0 6 votes vote down vote up
private void copyFileWithRetry(String description,
    FileStatus sourceFileStatus, Path target, Context context,
    FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes)
    throws IOException {
  long bytesCopied;
  try {
    bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description,
        action).execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
}
 
Example 3
Source Project: hadoop   Source File: CopyMapper.java    License: Apache License 2.0 6 votes vote down vote up
private boolean canSkip(FileSystem sourceFS, FileStatus source, 
    FileStatus target) throws IOException {
  if (!syncFolders) {
    return true;
  }
  boolean sameLength = target.getLen() == source.getLen();
  boolean sameBlockSize = source.getBlockSize() == target.getBlockSize()
      || !preserve.contains(FileAttribute.BLOCKSIZE);
  if (sameLength && sameBlockSize) {
    return skipCrc ||
        DistCpUtils.checksumsAreEqual(sourceFS, source.getPath(), null,
            targetFS, target.getPath());
  } else {
    return false;
  }
}
 
Example 4
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPackAttributes() {
  EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "");

  attributes.add(FileAttribute.REPLICATION);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "R");

  attributes.add(FileAttribute.BLOCKSIZE);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RB");

  attributes.add(FileAttribute.USER);
  attributes.add(FileAttribute.CHECKSUMTYPE);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUC");

  attributes.add(FileAttribute.GROUP);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGC");

  attributes.add(FileAttribute.PERMISSION);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPC");

  attributes.add(FileAttribute.TIMES);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPCT");
}
 
Example 5
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 6 votes vote down vote up
public void testUnpackAttributes() {
  EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RCBUGPAXT"));

  attributes.remove(FileAttribute.REPLICATION);
  attributes.remove(FileAttribute.CHECKSUMTYPE);
  attributes.remove(FileAttribute.ACL);
  attributes.remove(FileAttribute.XATTR);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGPT"));

  attributes.remove(FileAttribute.TIMES);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGP"));

  attributes.remove(FileAttribute.BLOCKSIZE);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UGP"));

  attributes.remove(FileAttribute.GROUP);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UP"));

  attributes.remove(FileAttribute.USER);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("P"));

  attributes.remove(FileAttribute.PERMISSION);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes(""));
}
 
Example 6
Source Project: big-c   Source File: RetriableFileCopyCommand.java    License: Apache License 2.0 6 votes vote down vote up
private long copyToFile(Path targetPath, FileSystem targetFS,
    FileStatus sourceFileStatus, long sourceOffset, Mapper.Context context,
    EnumSet<FileAttribute> fileAttributes, final FileChecksum sourceChecksum)
    throws IOException {
  FsPermission permission = FsPermission.getFileDefault().applyUMask(
      FsPermission.getUMask(targetFS.getConf()));
  final OutputStream outStream;
  if (action == FileAction.OVERWRITE) {
    final short repl = getReplicationFactor(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    final long blockSize = getBlockSize(fileAttributes, sourceFileStatus,
        targetFS, targetPath);
    FSDataOutputStream out = targetFS.create(targetPath, permission,
        EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
        BUFFER_SIZE, repl, blockSize, context,
        getChecksumOpt(fileAttributes, sourceChecksum));
    outStream = new BufferedOutputStream(out);
  } else {
    outStream = new BufferedOutputStream(targetFS.append(targetPath,
        BUFFER_SIZE));
  }
  return copyBytes(sourceFileStatus, sourceOffset, outStream, BUFFER_SIZE,
      context);
}
 
Example 7
Source Project: big-c   Source File: CopyMapper.java    License: Apache License 2.0 6 votes vote down vote up
private void copyFileWithRetry(String description,
    FileStatus sourceFileStatus, Path target, Context context,
    FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes)
    throws IOException {
  long bytesCopied;
  try {
    bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description,
        action).execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
}
 
Example 8
Source Project: big-c   Source File: CopyMapper.java    License: Apache License 2.0 6 votes vote down vote up
private boolean canSkip(FileSystem sourceFS, FileStatus source, 
    FileStatus target) throws IOException {
  if (!syncFolders) {
    return true;
  }
  boolean sameLength = target.getLen() == source.getLen();
  boolean sameBlockSize = source.getBlockSize() == target.getBlockSize()
      || !preserve.contains(FileAttribute.BLOCKSIZE);
  if (sameLength && sameBlockSize) {
    return skipCrc ||
        DistCpUtils.checksumsAreEqual(sourceFS, source.getPath(), null,
            targetFS, target.getPath());
  } else {
    return false;
  }
}
 
Example 9
Source Project: big-c   Source File: TestDistCpUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPackAttributes() {
  EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "");

  attributes.add(FileAttribute.REPLICATION);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "R");

  attributes.add(FileAttribute.BLOCKSIZE);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RB");

  attributes.add(FileAttribute.USER);
  attributes.add(FileAttribute.CHECKSUMTYPE);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUC");

  attributes.add(FileAttribute.GROUP);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGC");

  attributes.add(FileAttribute.PERMISSION);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPC");

  attributes.add(FileAttribute.TIMES);
  Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPCT");
}
 
Example 10
Source Project: big-c   Source File: TestDistCpUtils.java    License: Apache License 2.0 6 votes vote down vote up
public void testUnpackAttributes() {
  EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RCBUGPAXT"));

  attributes.remove(FileAttribute.REPLICATION);
  attributes.remove(FileAttribute.CHECKSUMTYPE);
  attributes.remove(FileAttribute.ACL);
  attributes.remove(FileAttribute.XATTR);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGPT"));

  attributes.remove(FileAttribute.TIMES);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGP"));

  attributes.remove(FileAttribute.BLOCKSIZE);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UGP"));

  attributes.remove(FileAttribute.GROUP);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UP"));

  attributes.remove(FileAttribute.USER);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("P"));

  attributes.remove(FileAttribute.PERMISSION);
  Assert.assertEquals(attributes, DistCpUtils.unpackAttributes(""));
}
 
Example 11
Source Project: circus-train   Source File: DistCpOptionsParserTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void typical() {
  Map<String, Object> options = defaultOptions();
  options.put(FILE_ATTRIBUTES, Arrays.asList("replication", "blocksize", "user", "group", "permission",
      "checksumtype", "acl", "xattr", "times"));
  options.put(PRESERVE_RAW_XATTRS, "true");
  options.put(ATOMIC_COMMIT, "false");
  options.put(ATOMIC_WORK_PATH, "atomic-work-path");
  options.put(COPY_STRATEGY, "copy-strategy");
  options.put(IGNORE_FAILURES, "true");
  options.put(LOG_PATH, "log-path");
  options.put(TASK_BANDWIDTH, "500");
  options.put(MAX_MAPS, "2");
  options.put(SKIP_CRC, "false");
  options.put(SSL_CONFIGURATION_FILE, "ssl-configuration-file");
  DistCpOptions distCpOptions = parser.parse(options);
  for (FileAttribute attribute : FileAttribute.values()) {
    assertThat(distCpOptions.shouldPreserve(attribute), is(true));
  }
  assertThat(distCpOptions.shouldPreserveRawXattrs(), is(true));
  assertThat(distCpOptions.shouldAppend(), is(false));
  assertThat(distCpOptions.shouldAtomicCommit(), is(false));
  assertThat(distCpOptions.getAtomicWorkPath(), is(new Path("atomic-work-path")));
  assertThat(distCpOptions.shouldBlock(), is(true));
  assertThat(distCpOptions.getCopyStrategy(), is("copy-strategy"));
  assertThat(distCpOptions.shouldDeleteMissing(), is(false));
  assertThat(distCpOptions.shouldIgnoreFailures(), is(true));
  assertThat(distCpOptions.getLogPath(), is(new Path("log-path")));
  assertThat(distCpOptions.getMapBandwidth(), is(500));
  assertThat(distCpOptions.getMaxMaps(), is(2));
  assertThat(distCpOptions.shouldOverwrite(), is(false));
  assertThat(distCpOptions.shouldSkipCRC(), is(false));
  assertThat(distCpOptions.getSslConfigurationFile(), is("ssl-configuration-file"));
  assertThat(distCpOptions.shouldSyncFolder(), is(false));
  assertThat(distCpOptions.getTargetPathExists(), is(true));
}
 
Example 12
Source Project: hadoop   Source File: SimpleCopyListing.java    License: Apache License 2.0 5 votes vote down vote up
private void traverseNonEmptyDirectory(SequenceFile.Writer fileListWriter,
                                       FileStatus sourceStatus,
                                       Path sourcePathRoot,
                                       DistCpOptions options)
                                       throws IOException {
  FileSystem sourceFS = sourcePathRoot.getFileSystem(getConf());
  final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
  final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
  final boolean preserveRawXattrs = options.shouldPreserveRawXattrs();
  Stack<FileStatus> pathStack = new Stack<FileStatus>();
  pathStack.push(sourceStatus);

  while (!pathStack.isEmpty()) {
    for (FileStatus child: getChildren(sourceFS, pathStack.pop())) {
      if (LOG.isDebugEnabled())
        LOG.debug("Recording source-path: "
                  + sourceStatus.getPath() + " for copy.");
      CopyListingFileStatus childCopyListingStatus =
        DistCpUtils.toCopyListingFileStatus(sourceFS, child,
          preserveAcls && child.isDirectory(),
          preserveXAttrs && child.isDirectory(),
          preserveRawXattrs && child.isDirectory());
      writeToFileListing(fileListWriter, childCopyListingStatus,
           sourcePathRoot, options);
      if (isDirectoryAndNotEmpty(sourceFS, child)) {
        if (LOG.isDebugEnabled())
          LOG.debug("Traversing non-empty source dir: "
                     + sourceStatus.getPath());
        pathStack.push(child);
      }
    }
  }
}
 
Example 13
Source Project: hadoop   Source File: RetriableFileCopyCommand.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return the checksum spec of the source checksum if checksum type should be
 *         preserved
 */
private ChecksumOpt getChecksumOpt(EnumSet<FileAttribute> fileAttributes,
    FileChecksum sourceChecksum) {
  if (fileAttributes.contains(FileAttribute.CHECKSUMTYPE)
      && sourceChecksum != null) {
    return sourceChecksum.getChecksumOpt();
  }
  return null;
}
 
Example 14
Source Project: hadoop   Source File: RetriableFileCopyCommand.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return the block size of the source file if we need to preserve either
 *         the block size or the checksum type. Otherwise the default block
 *         size of the target FS.
 */
private static long getBlockSize(
        EnumSet<FileAttribute> fileAttributes,
        FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
  boolean preserve = fileAttributes.contains(FileAttribute.BLOCKSIZE)
      || fileAttributes.contains(FileAttribute.CHECKSUMTYPE);
  return preserve ? sourceFile.getBlockSize() : targetFS
      .getDefaultBlockSize(tmpTargetPath);
}
 
Example 15
Source Project: hadoop   Source File: DistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Pack file preservation attributes into a string, containing
 * just the first character of each preservation attribute
 * @param attributes - Attribute set to preserve
 * @return - String containing first letters of each attribute to preserve
 */
public static String packAttributes(EnumSet<FileAttribute> attributes) {
  StringBuffer buffer = new StringBuffer(FileAttribute.values().length);
  int len = 0;
  for (FileAttribute attribute : attributes) {
    buffer.append(attribute.name().charAt(0));
    len++;
  }
  return buffer.substring(0, len);
}
 
Example 16
Source Project: hadoop   Source File: DistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Unpacks preservation attribute string containing the first character of
 * each preservation attribute back to a set of attributes to preserve
 * @param attributes - Attribute string
 * @return - Attribute set
 */
public static EnumSet<FileAttribute> unpackAttributes(String attributes) {
  EnumSet<FileAttribute> retValue = EnumSet.noneOf(FileAttribute.class);

  if (attributes != null) {
    for (int index = 0; index < attributes.length(); index++) {
      retValue.add(FileAttribute.getAttribute(attributes.charAt(index)));
    }
  }

  return retValue;
}
 
Example 17
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveDefaults() throws IOException {
  FileSystem fs = FileSystem.get(config);
  
  // preserve replication, block size, user, group, permission, 
  // checksum type and timestamps    
  EnumSet<FileAttribute> attributes = 
      DistCpUtils.unpackAttributes(
          DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.substring(1));

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);
  
  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 18
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveNothingOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(dstStatus.getAccessTime() == 100);
  Assert.assertTrue(dstStatus.getModificationTime() == 100);
  Assert.assertTrue(dstStatus.getReplication() == 0);
}
 
Example 19
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreservePermissionOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
}
 
Example 20
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveGroupOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
}
 
Example 21
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveUserOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
}
 
Example 22
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveReplicationOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  // Replication shouldn't apply to dirs so this should still be 0 == 0
  Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 23
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveTimestampOnDirectory() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);

  Path dst = new Path("/tmp/abc");
  Path src = new Path("/tmp/src");

  createDirectory(fs, src);
  createDirectory(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
}
 
Example 24
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveNothingOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 25
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreservePermissionOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 26
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveGroupOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 27
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveUserOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 28
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveReplicationOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 29
Source Project: hadoop   Source File: TestDistCpUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreserveTimestampOnFile() throws IOException {
  FileSystem fs = FileSystem.get(config);
  EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES);

  Path dst = new Path("/tmp/dest2");
  Path src = new Path("/tmp/src2");

  createFile(fs, src);
  createFile(fs, dst);

  fs.setPermission(src, fullPerm);
  fs.setOwner(src, "somebody", "somebody-group");
  fs.setTimes(src, 0, 0);
  fs.setReplication(src, (short) 1);

  fs.setPermission(dst, noPerm);
  fs.setOwner(dst, "nobody", "nobody-group");
  fs.setTimes(dst, 100, 100);
  fs.setReplication(dst, (short) 2);

  CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src));

  DistCpUtils.preserve(fs, dst, srcStatus, attributes, false);

  CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst));

  // FileStatus.equals only compares path field, must explicitly compare all fields
  Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission()));
  Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner()));
  Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup()));
  Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime());
  Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime());
  Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication());
}
 
Example 30
Source Project: big-c   Source File: SimpleCopyListing.java    License: Apache License 2.0 5 votes vote down vote up
private void traverseNonEmptyDirectory(SequenceFile.Writer fileListWriter,
                                       FileStatus sourceStatus,
                                       Path sourcePathRoot,
                                       DistCpOptions options)
                                       throws IOException {
  FileSystem sourceFS = sourcePathRoot.getFileSystem(getConf());
  final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
  final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
  final boolean preserveRawXattrs = options.shouldPreserveRawXattrs();
  Stack<FileStatus> pathStack = new Stack<FileStatus>();
  pathStack.push(sourceStatus);

  while (!pathStack.isEmpty()) {
    for (FileStatus child: getChildren(sourceFS, pathStack.pop())) {
      if (LOG.isDebugEnabled())
        LOG.debug("Recording source-path: "
                  + sourceStatus.getPath() + " for copy.");
      CopyListingFileStatus childCopyListingStatus =
        DistCpUtils.toCopyListingFileStatus(sourceFS, child,
          preserveAcls && child.isDirectory(),
          preserveXAttrs && child.isDirectory(),
          preserveRawXattrs && child.isDirectory());
      writeToFileListing(fileListWriter, childCopyListingStatus,
           sourcePathRoot, options);
      if (isDirectoryAndNotEmpty(sourceFS, child)) {
        if (LOG.isDebugEnabled())
          LOG.debug("Traversing non-empty source dir: "
                     + sourceStatus.getPath());
        pathStack.push(child);
      }
    }
  }
}