Java Code Examples for org.apache.hadoop.tools.DistCpOptions#FileAttribute

The following examples show how to use org.apache.hadoop.tools.DistCpOptions#FileAttribute . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: CopyMapper.java From hadoop with Apache License 2.0

6 votes

private void copyFileWithRetry(String description,
    FileStatus sourceFileStatus, Path target, Context context,
    FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes)
    throws IOException {
  long bytesCopied;
  try {
    bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description,
        action).execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
}

Example 2

Source File: CopyMapper.java From big-c with Apache License 2.0

6 votes

private void copyFileWithRetry(String description,
    FileStatus sourceFileStatus, Path target, Context context,
    FileAction action, EnumSet<DistCpOptions.FileAttribute> fileAttributes)
    throws IOException {
  long bytesCopied;
  try {
    bytesCopied = (Long) new RetriableFileCopyCommand(skipCrc, description,
        action).execute(sourceFileStatus, target, context, fileAttributes);
  } catch (Exception e) {
    context.setStatus("Copy Failure: " + sourceFileStatus.getPath());
    throw new IOException("File copy failed: " + sourceFileStatus.getPath() +
        " --> " + target, e);
  }
  incrementCounter(context, Counter.BYTESEXPECTED, sourceFileStatus.getLen());
  incrementCounter(context, Counter.BYTESCOPIED, bytesCopied);
  incrementCounter(context, Counter.COPY, 1);
}

Example 3

Source File: TestCopyMapper.java From hadoop with Apache License 2.0

5 votes

@Test(timeout=40000)
public void testCopyFailOnBlockSizeDifference() {
  try {

    deleteState();
    createSourceDataWithDifferentBlockSize();

    FileSystem fs = cluster.getFileSystem();
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
        = stubContext.getContext();

    Configuration configuration = context.getConfiguration();
    EnumSet<DistCpOptions.FileAttribute> fileAttributes
        = EnumSet.noneOf(DistCpOptions.FileAttribute.class);
    configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
        DistCpUtils.packAttributes(fileAttributes));

    copyMapper.setup(context);

    for (Path path : pathList) {
      final FileStatus fileStatus = fs.getFileStatus(path);
      copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
          new CopyListingFileStatus(fileStatus), context);
    }

    Assert.fail("Copy should have failed because of block-size difference.");
  }
  catch (Exception exception) {
    // Check that the exception suggests the use of -pb/-skipCrc.
    Assert.assertTrue("Failure exception should have suggested the use of -pb.", exception.getCause().getCause().getMessage().contains("pb"));
    Assert.assertTrue("Failure exception should have suggested the use of -skipCrc.", exception.getCause().getCause().getMessage().contains("skipCrc"));
  }
}

Example 4

Source File: TestCopyMapper.java From big-c with Apache License 2.0

5 votes

@Test(timeout=40000)
public void testCopyFailOnBlockSizeDifference() {
  try {

    deleteState();
    createSourceDataWithDifferentBlockSize();

    FileSystem fs = cluster.getFileSystem();
    CopyMapper copyMapper = new CopyMapper();
    StubContext stubContext = new StubContext(getConfiguration(), null, 0);
    Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
        = stubContext.getContext();

    Configuration configuration = context.getConfiguration();
    EnumSet<DistCpOptions.FileAttribute> fileAttributes
        = EnumSet.noneOf(DistCpOptions.FileAttribute.class);
    configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
        DistCpUtils.packAttributes(fileAttributes));

    copyMapper.setup(context);

    for (Path path : pathList) {
      final FileStatus fileStatus = fs.getFileStatus(path);
      copyMapper.map(new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
          new CopyListingFileStatus(fileStatus), context);
    }

    Assert.fail("Copy should have failed because of block-size difference.");
  }
  catch (Exception exception) {
    // Check that the exception suggests the use of -pb/-skipCrc.
    Assert.assertTrue("Failure exception should have suggested the use of -pb.", exception.getCause().getCause().getMessage().contains("pb"));
    Assert.assertTrue("Failure exception should have suggested the use of -skipCrc.", exception.getCause().getCause().getMessage().contains("skipCrc"));
  }
}

Example 5

Source File: CopyMapper.java From hadoop with Apache License 2.0

4 votes

/**
 * Implementation of the Mapper::map(). Does the copy.
 * @param relPath The target path.
 * @param sourceFileStatus The source path.
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public void map(Text relPath, CopyListingFileStatus sourceFileStatus,
        Context context) throws IOException, InterruptedException {
  Path sourcePath = sourceFileStatus.getPath();

  if (LOG.isDebugEnabled())
    LOG.debug("DistCpMapper::map(): Received " + sourcePath + ", " + relPath);

  Path target = new Path(targetWorkPath.makeQualified(targetFS.getUri(),
                        targetFS.getWorkingDirectory()) + relPath.toString());

  EnumSet<DistCpOptions.FileAttribute> fileAttributes
          = getFileAttributeSettings(context);
  final boolean preserveRawXattrs = context.getConfiguration().getBoolean(
      DistCpConstants.CONF_LABEL_PRESERVE_RAWXATTRS, false);

  final String description = "Copying " + sourcePath + " to " + target;
  context.setStatus(description);

  LOG.info(description);

  try {
    CopyListingFileStatus sourceCurrStatus;
    FileSystem sourceFS;
    try {
      sourceFS = sourcePath.getFileSystem(conf);
      final boolean preserveXAttrs =
          fileAttributes.contains(FileAttribute.XATTR);
      sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
        sourceFS.getFileStatus(sourcePath),
        fileAttributes.contains(FileAttribute.ACL), 
        preserveXAttrs, preserveRawXattrs);
    } catch (FileNotFoundException e) {
      throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
    }

    FileStatus targetStatus = null;

    try {
      targetStatus = targetFS.getFileStatus(target);
    } catch (FileNotFoundException ignore) {
      if (LOG.isDebugEnabled())
        LOG.debug("Path could not be found: " + target, ignore);
    }

    if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
      throw new IOException("Can't replace " + target + ". Target is " +
          getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
    }

    if (sourceCurrStatus.isDirectory()) {
      createTargetDirsWithRetry(description, target, context);
      return;
    }

    FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target);
    if (action == FileAction.SKIP) {
      LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
               + " to " + target);
      updateSkipCounters(context, sourceCurrStatus);
      context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
    } else {
      copyFileWithRetry(description, sourceCurrStatus, target, context,
          action, fileAttributes);
    }

    DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus,
        fileAttributes, preserveRawXattrs);
  } catch (IOException exception) {
    handleFailures(exception, sourceFileStatus, target, context);
  }
}

Example 6

Source File: CopyMapper.java From hadoop with Apache License 2.0

4 votes

private static EnumSet<DistCpOptions.FileAttribute>
        getFileAttributeSettings(Mapper.Context context) {
  String attributeString = context.getConfiguration().get(
          DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel());
  return DistCpUtils.unpackAttributes(attributeString);
}

Example 7

Source File: TestCopyMapper.java From hadoop with Apache License 2.0

4 votes

private void testCopy(boolean preserveChecksum) throws Exception {
  deleteState();
  if (preserveChecksum) {
    createSourceDataWithDifferentChecksumType();
  } else {
    createSourceData();
  }

  FileSystem fs = cluster.getFileSystem();
  CopyMapper copyMapper = new CopyMapper();
  StubContext stubContext = new StubContext(getConfiguration(), null, 0);
  Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
          = stubContext.getContext();

  Configuration configuration = context.getConfiguration();
  EnumSet<DistCpOptions.FileAttribute> fileAttributes
          = EnumSet.of(DistCpOptions.FileAttribute.REPLICATION);
  if (preserveChecksum) {
    fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE);
  }
  configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
          DistCpUtils.packAttributes(fileAttributes));

  copyMapper.setup(context);

  for (Path path: pathList) {
    copyMapper.map(
        new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
        new CopyListingFileStatus(fs.getFileStatus(path)), context);
  }

  // Check that the maps worked.
  verifyCopy(fs, preserveChecksum);
  Assert.assertEquals(pathList.size(), stubContext.getReporter()
      .getCounter(CopyMapper.Counter.COPY).getValue());
  if (!preserveChecksum) {
    Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE, stubContext
        .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED)
        .getValue());
  } else {
    Assert.assertEquals(nFiles * NON_DEFAULT_BLOCK_SIZE * 2, stubContext
        .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED)
        .getValue());
  }

  testCopyingExistingFiles(fs, copyMapper, context);
  for (Text value : stubContext.getWriter().values()) {
    Assert.assertTrue(value.toString() + " is not skipped", value
        .toString().startsWith("SKIP:"));
  }
}

Example 8

Source File: CopyMapper.java From big-c with Apache License 2.0

4 votes

/**
 * Implementation of the Mapper::map(). Does the copy.
 * @param relPath The target path.
 * @param sourceFileStatus The source path.
 * @throws IOException
 * @throws InterruptedException
 */
@Override
public void map(Text relPath, CopyListingFileStatus sourceFileStatus,
        Context context) throws IOException, InterruptedException {
  Path sourcePath = sourceFileStatus.getPath();

  if (LOG.isDebugEnabled())
    LOG.debug("DistCpMapper::map(): Received " + sourcePath + ", " + relPath);

  Path target = new Path(targetWorkPath.makeQualified(targetFS.getUri(),
                        targetFS.getWorkingDirectory()) + relPath.toString());

  EnumSet<DistCpOptions.FileAttribute> fileAttributes
          = getFileAttributeSettings(context);
  final boolean preserveRawXattrs = context.getConfiguration().getBoolean(
      DistCpConstants.CONF_LABEL_PRESERVE_RAWXATTRS, false);

  final String description = "Copying " + sourcePath + " to " + target;
  context.setStatus(description);

  LOG.info(description);

  try {
    CopyListingFileStatus sourceCurrStatus;
    FileSystem sourceFS;
    try {
      sourceFS = sourcePath.getFileSystem(conf);
      final boolean preserveXAttrs =
          fileAttributes.contains(FileAttribute.XATTR);
      sourceCurrStatus = DistCpUtils.toCopyListingFileStatus(sourceFS,
        sourceFS.getFileStatus(sourcePath),
        fileAttributes.contains(FileAttribute.ACL), 
        preserveXAttrs, preserveRawXattrs);
    } catch (FileNotFoundException e) {
      throw new IOException(new RetriableFileCopyCommand.CopyReadException(e));
    }

    FileStatus targetStatus = null;

    try {
      targetStatus = targetFS.getFileStatus(target);
    } catch (FileNotFoundException ignore) {
      if (LOG.isDebugEnabled())
        LOG.debug("Path could not be found: " + target, ignore);
    }

    if (targetStatus != null && (targetStatus.isDirectory() != sourceCurrStatus.isDirectory())) {
      throw new IOException("Can't replace " + target + ". Target is " +
          getFileType(targetStatus) + ", Source is " + getFileType(sourceCurrStatus));
    }

    if (sourceCurrStatus.isDirectory()) {
      createTargetDirsWithRetry(description, target, context);
      return;
    }

    FileAction action = checkUpdate(sourceFS, sourceCurrStatus, target);
    if (action == FileAction.SKIP) {
      LOG.info("Skipping copy of " + sourceCurrStatus.getPath()
               + " to " + target);
      updateSkipCounters(context, sourceCurrStatus);
      context.write(null, new Text("SKIP: " + sourceCurrStatus.getPath()));
    } else {
      copyFileWithRetry(description, sourceCurrStatus, target, context,
          action, fileAttributes);
    }

    DistCpUtils.preserve(target.getFileSystem(conf), target, sourceCurrStatus,
        fileAttributes, preserveRawXattrs);
  } catch (IOException exception) {
    handleFailures(exception, sourceFileStatus, target, context);
  }
}

Example 9

Source File: CopyMapper.java From big-c with Apache License 2.0

4 votes

private static EnumSet<DistCpOptions.FileAttribute>
        getFileAttributeSettings(Mapper.Context context) {
  String attributeString = context.getConfiguration().get(
          DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel());
  return DistCpUtils.unpackAttributes(attributeString);
}

Example 10

Source File: TestCopyMapper.java From big-c with Apache License 2.0

4 votes

private void testCopy(boolean preserveChecksum) throws Exception {
  deleteState();
  if (preserveChecksum) {
    createSourceDataWithDifferentChecksumType();
  } else {
    createSourceData();
  }

  FileSystem fs = cluster.getFileSystem();
  CopyMapper copyMapper = new CopyMapper();
  StubContext stubContext = new StubContext(getConfiguration(), null, 0);
  Mapper<Text, CopyListingFileStatus, Text, Text>.Context context
          = stubContext.getContext();

  Configuration configuration = context.getConfiguration();
  EnumSet<DistCpOptions.FileAttribute> fileAttributes
          = EnumSet.of(DistCpOptions.FileAttribute.REPLICATION);
  if (preserveChecksum) {
    fileAttributes.add(DistCpOptions.FileAttribute.CHECKSUMTYPE);
  }
  configuration.set(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel(),
          DistCpUtils.packAttributes(fileAttributes));

  copyMapper.setup(context);

  for (Path path: pathList) {
    copyMapper.map(
        new Text(DistCpUtils.getRelativePath(new Path(SOURCE_PATH), path)),
        new CopyListingFileStatus(fs.getFileStatus(path)), context);
  }

  // Check that the maps worked.
  verifyCopy(fs, preserveChecksum);
  Assert.assertEquals(pathList.size(), stubContext.getReporter()
      .getCounter(CopyMapper.Counter.COPY).getValue());
  if (!preserveChecksum) {
    Assert.assertEquals(nFiles * DEFAULT_FILE_SIZE, stubContext
        .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED)
        .getValue());
  } else {
    Assert.assertEquals(nFiles * NON_DEFAULT_BLOCK_SIZE * 2, stubContext
        .getReporter().getCounter(CopyMapper.Counter.BYTESCOPIED)
        .getValue());
  }

  testCopyingExistingFiles(fs, copyMapper, context);
  for (Text value : stubContext.getWriter().values()) {
    Assert.assertTrue(value.toString() + " is not skipped", value
        .toString().startsWith("SKIP:"));
  }
}