Java Code Examples for org.apache.hadoop.fs.Options.CreateOpts#getOpt()

The following examples show how to use org.apache.hadoop.fs.Options.CreateOpts#getOpt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FileContextTestWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public long createFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out =
    fc.create(path, EnumSet.of(CreateFlag.CREATE), options);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
  return data.length;
}
Example 2
Source File: FileContextTestWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void appendToFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out;
  out = fc.create(path, EnumSet.of(CreateFlag.APPEND));
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
}
Example 3
Source File: FileContextTestHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static long createFile(FileContext fc, Path path, int numBlocks,
    CreateOpts... options) throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out = 
    fc.create(path, EnumSet.of(CreateFlag.CREATE), options);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
  return data.length;
}
Example 4
Source File: FileContextTestHelper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static void appendToFile(FileContext fc, Path path, int numBlocks,
    CreateOpts... options) throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out;
  out = fc.create(path, EnumSet.of(CreateFlag.APPEND));
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
}
Example 5
Source File: FileSystemTestWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public long createFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out =
    create(path, EnumSet.of(CreateFlag.CREATE), options);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
  return data.length;
}
Example 6
Source File: FileSystemTestWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public void appendToFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out;
  out = fs.append(path);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
}
Example 7
Source File: FileContextTestWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public long createFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out =
    fc.create(path, EnumSet.of(CreateFlag.CREATE), options);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
  return data.length;
}
Example 8
Source File: FileContextTestWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void appendToFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out;
  out = fc.create(path, EnumSet.of(CreateFlag.APPEND));
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
}
Example 9
Source File: FileContextTestHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static long createFile(FileContext fc, Path path, int numBlocks,
    CreateOpts... options) throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out = 
    fc.create(path, EnumSet.of(CreateFlag.CREATE), options);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
  return data.length;
}
Example 10
Source File: FileContextTestHelper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static void appendToFile(FileContext fc, Path path, int numBlocks,
    CreateOpts... options) throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out;
  out = fc.create(path, EnumSet.of(CreateFlag.APPEND));
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
}
Example 11
Source File: FileSystemTestWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public long createFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out =
    create(path, EnumSet.of(CreateFlag.CREATE), options);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
  return data.length;
}
Example 12
Source File: FileSystemTestWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public void appendToFile(Path path, int numBlocks, CreateOpts... options)
    throws IOException {
  BlockSize blockSizeOpt = CreateOpts.getOpt(CreateOpts.BlockSize.class, options);
  long blockSize = blockSizeOpt != null ? blockSizeOpt.getValue()
      : DEFAULT_BLOCK_SIZE;
  FSDataOutputStream out;
  out = fs.append(path);
  byte[] data = getFileData(numBlocks, blockSize);
  out.write(data, 0, data.length);
  out.close();
}
Example 13
Source File: FileSystemTestWrapper.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override
public FSDataOutputStream create(Path f, EnumSet<CreateFlag> createFlag,
    CreateOpts... opts) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException, IOException {

  // Need to translate the FileContext-style options into FileSystem-style

  // Permissions with umask
  CreateOpts.Perms permOpt = CreateOpts.getOpt(
      CreateOpts.Perms.class, opts);
  FsPermission umask = FsPermission.getUMask(fs.getConf());
  FsPermission permission = (permOpt != null) ? permOpt.getValue()
      : FsPermission.getFileDefault().applyUMask(umask);
  permission = permission.applyUMask(umask);
  // Overwrite
  boolean overwrite = createFlag.contains(CreateFlag.OVERWRITE);
  // bufferSize
  int bufferSize = fs.getConf().getInt(
      CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
      CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
  CreateOpts.BufferSize bufOpt = CreateOpts.getOpt(
      CreateOpts.BufferSize.class, opts);
  bufferSize = (bufOpt != null) ? bufOpt.getValue() : bufferSize;
  // replication
  short replication = fs.getDefaultReplication(f);
  CreateOpts.ReplicationFactor repOpt =
      CreateOpts.getOpt(CreateOpts.ReplicationFactor.class, opts);
  replication = (repOpt != null) ? repOpt.getValue() : replication;
  // blockSize
  long blockSize = fs.getDefaultBlockSize(f);
  CreateOpts.BlockSize blockOpt = CreateOpts.getOpt(
      CreateOpts.BlockSize.class, opts);
  blockSize = (blockOpt != null) ? blockOpt.getValue() : blockSize;
  // Progressable
  Progressable progress = null;
  CreateOpts.Progress progressOpt = CreateOpts.getOpt(
      CreateOpts.Progress.class, opts);
  progress = (progressOpt != null) ? progressOpt.getValue() : progress;
  return fs.create(f, permission, overwrite, bufferSize, replication,
      blockSize, progress);
}
Example 14
Source File: FileSystemTestWrapper.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override
public FSDataOutputStream create(Path f, EnumSet<CreateFlag> createFlag,
    CreateOpts... opts) throws AccessControlException,
    FileAlreadyExistsException, FileNotFoundException,
    ParentNotDirectoryException, UnsupportedFileSystemException, IOException {

  // Need to translate the FileContext-style options into FileSystem-style

  // Permissions with umask
  CreateOpts.Perms permOpt = CreateOpts.getOpt(
      CreateOpts.Perms.class, opts);
  FsPermission umask = FsPermission.getUMask(fs.getConf());
  FsPermission permission = (permOpt != null) ? permOpt.getValue()
      : FsPermission.getFileDefault().applyUMask(umask);
  permission = permission.applyUMask(umask);
  // Overwrite
  boolean overwrite = createFlag.contains(CreateFlag.OVERWRITE);
  // bufferSize
  int bufferSize = fs.getConf().getInt(
      CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_KEY,
      CommonConfigurationKeysPublic.IO_FILE_BUFFER_SIZE_DEFAULT);
  CreateOpts.BufferSize bufOpt = CreateOpts.getOpt(
      CreateOpts.BufferSize.class, opts);
  bufferSize = (bufOpt != null) ? bufOpt.getValue() : bufferSize;
  // replication
  short replication = fs.getDefaultReplication(f);
  CreateOpts.ReplicationFactor repOpt =
      CreateOpts.getOpt(CreateOpts.ReplicationFactor.class, opts);
  replication = (repOpt != null) ? repOpt.getValue() : replication;
  // blockSize
  long blockSize = fs.getDefaultBlockSize(f);
  CreateOpts.BlockSize blockOpt = CreateOpts.getOpt(
      CreateOpts.BlockSize.class, opts);
  blockSize = (blockOpt != null) ? blockOpt.getValue() : blockSize;
  // Progressable
  Progressable progress = null;
  CreateOpts.Progress progressOpt = CreateOpts.getOpt(
      CreateOpts.Progress.class, opts);
  progress = (progressOpt != null) ? progressOpt.getValue() : progress;
  return fs.create(f, permission, overwrite, bufferSize, replication,
      blockSize, progress);
}
Example 15
Source File: FileContext.java    From hadoop with Apache License 2.0 3 votes vote down vote up 
/**
 * Create or overwrite file on indicated path and returns an output stream for
 * writing into the file.
 * 
 * @param f the file name to open
 * @param createFlag gives the semantics of create; see {@link CreateFlag}
 * @param opts file creation options; see {@link Options.CreateOpts}.
 *          <ul>
 *          <li>Progress - to report progress on the operation - default null
 *          <li>Permission - umask is applied against permisssion: default is
 *          FsPermissions:getDefault()
 * 
 *          <li>CreateParent - create missing parent path; default is to not
 *          to create parents
 *          <li>The defaults for the following are SS defaults of the file
 *          server implementing the target path. Not all parameters make sense
 *          for all kinds of file system - eg. localFS ignores Blocksize,
 *          replication, checksum
 *          <ul>
 *          <li>BufferSize - buffersize used in FSDataOutputStream
 *          <li>Blocksize - block size for file blocks
 *          <li>ReplicationFactor - replication for blocks
 *          <li>ChecksumParam - Checksum parameters. server default is used
 *          if not specified.
 *          </ul>
 *          </ul>
 * 
 * @return {@link FSDataOutputStream} for created file
 * 
 * @throws AccessControlException If access is denied
 * @throws FileAlreadyExistsException If file <code>f</code> already exists
 * @throws FileNotFoundException If parent of <code>f</code> does not exist
 *           and <code>createParent</code> is false
 * @throws ParentNotDirectoryException If parent of <code>f</code> is not a
 *           directory.
 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
 *           not supported
 * @throws IOException If an I/O error occurred
 * 
 * Exceptions applicable to file systems accessed over RPC:
 * @throws RpcClientException If an exception occurred in the RPC client
 * @throws RpcServerException If an exception occurred in the RPC server
 * @throws UnexpectedServerException If server implementation throws
 *           undeclared exception to RPC server
 * 
 * RuntimeExceptions:
 * @throws InvalidPathException If path <code>f</code> is not valid
 */
public FSDataOutputStream create(final Path f,
    final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
    throws AccessControlException, FileAlreadyExistsException,
    FileNotFoundException, ParentNotDirectoryException,
    UnsupportedFileSystemException, IOException {
  Path absF = fixRelativePart(f);

  // If one of the options is a permission, extract it & apply umask
  // If not, add a default Perms and apply umask;
  // AbstractFileSystem#create

  CreateOpts.Perms permOpt = CreateOpts.getOpt(CreateOpts.Perms.class, opts);
  FsPermission permission = (permOpt != null) ? permOpt.getValue() :
                                    FILE_DEFAULT_PERM;
  permission = permission.applyUMask(umask);

  final CreateOpts[] updatedOpts = 
                    CreateOpts.setOpt(CreateOpts.perms(permission), opts);
  return new FSLinkResolver<FSDataOutputStream>() {
    @Override
    public FSDataOutputStream next(final AbstractFileSystem fs, final Path p) 
      throws IOException {
      return fs.create(p, createFlag, updatedOpts);
    }
  }.resolve(this, absF);
}
Example 16
Source File: FileContext.java    From big-c with Apache License 2.0 3 votes vote down vote up 
/**
 * Create or overwrite file on indicated path and returns an output stream for
 * writing into the file.
 * 
 * @param f the file name to open
 * @param createFlag gives the semantics of create; see {@link CreateFlag}
 * @param opts file creation options; see {@link Options.CreateOpts}.
 *          <ul>
 *          <li>Progress - to report progress on the operation - default null
 *          <li>Permission - umask is applied against permisssion: default is
 *          FsPermissions:getDefault()
 * 
 *          <li>CreateParent - create missing parent path; default is to not
 *          to create parents
 *          <li>The defaults for the following are SS defaults of the file
 *          server implementing the target path. Not all parameters make sense
 *          for all kinds of file system - eg. localFS ignores Blocksize,
 *          replication, checksum
 *          <ul>
 *          <li>BufferSize - buffersize used in FSDataOutputStream
 *          <li>Blocksize - block size for file blocks
 *          <li>ReplicationFactor - replication for blocks
 *          <li>ChecksumParam - Checksum parameters. server default is used
 *          if not specified.
 *          </ul>
 *          </ul>
 * 
 * @return {@link FSDataOutputStream} for created file
 * 
 * @throws AccessControlException If access is denied
 * @throws FileAlreadyExistsException If file <code>f</code> already exists
 * @throws FileNotFoundException If parent of <code>f</code> does not exist
 *           and <code>createParent</code> is false
 * @throws ParentNotDirectoryException If parent of <code>f</code> is not a
 *           directory.
 * @throws UnsupportedFileSystemException If file system for <code>f</code> is
 *           not supported
 * @throws IOException If an I/O error occurred
 * 
 * Exceptions applicable to file systems accessed over RPC:
 * @throws RpcClientException If an exception occurred in the RPC client
 * @throws RpcServerException If an exception occurred in the RPC server
 * @throws UnexpectedServerException If server implementation throws
 *           undeclared exception to RPC server
 * 
 * RuntimeExceptions:
 * @throws InvalidPathException If path <code>f</code> is not valid
 */
public FSDataOutputStream create(final Path f,
    final EnumSet<CreateFlag> createFlag, Options.CreateOpts... opts)
    throws AccessControlException, FileAlreadyExistsException,
    FileNotFoundException, ParentNotDirectoryException,
    UnsupportedFileSystemException, IOException {
  Path absF = fixRelativePart(f);

  // If one of the options is a permission, extract it & apply umask
  // If not, add a default Perms and apply umask;
  // AbstractFileSystem#create

  CreateOpts.Perms permOpt = CreateOpts.getOpt(CreateOpts.Perms.class, opts);
  FsPermission permission = (permOpt != null) ? permOpt.getValue() :
                                    FILE_DEFAULT_PERM;
  permission = permission.applyUMask(umask);

  final CreateOpts[] updatedOpts = 
                    CreateOpts.setOpt(CreateOpts.perms(permission), opts);
  return new FSLinkResolver<FSDataOutputStream>() {
    @Override
    public FSDataOutputStream next(final AbstractFileSystem fs, final Path p) 
      throws IOException {
      return fs.create(p, createFlag, updatedOpts);
    }
  }.resolve(this, absF);
}