Java Code Examples for org.apache.hadoop.fs.FileSystem#getDefaultReplication()

The following examples show how to use org.apache.hadoop.fs.FileSystem#getDefaultReplication() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: FSOperations.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Executes the filesystem operation.
 *
 * @param fs filesystem instance to use.
 *
 * @return The URI of the created file.
 *
 * @throws IOException thrown if an IO error occured.
 */
@Override
public Void execute(FileSystem fs) throws IOException {
  if (replication == -1) {
    replication = fs.getDefaultReplication(path);
  }
  if (blockSize == -1) {
    blockSize = fs.getDefaultBlockSize(path);
  }
  FsPermission fsPermission = new FsPermission(permission);
  int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
  OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
  IOUtils.copyBytes(is, os, bufferSize, true);
  os.close();
  return null;
}
 
Example 2
Source Project: big-c   File: FSOperations.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Executes the filesystem operation.
 *
 * @param fs filesystem instance to use.
 *
 * @return The URI of the created file.
 *
 * @throws IOException thrown if an IO error occured.
 */
@Override
public Void execute(FileSystem fs) throws IOException {
  if (replication == -1) {
    replication = fs.getDefaultReplication(path);
  }
  if (blockSize == -1) {
    blockSize = fs.getDefaultBlockSize(path);
  }
  FsPermission fsPermission = new FsPermission(permission);
  int bufferSize = fs.getConf().getInt("httpfs.buffer.size", 4096);
  OutputStream os = fs.create(path, fsPermission, override, bufferSize, replication, blockSize, null);
  IOUtils.copyBytes(is, os, bufferSize, true);
  os.close();
  return null;
}
 
Example 3
Source Project: lucene-solr   File: HdfsFileWriter.java    License: Apache License 2.0 5 votes vote down vote up
private static final OutputStream getOutputStream(FileSystem fileSystem, Path path) throws IOException {
  Configuration conf = fileSystem.getConf();
  FsServerDefaults fsDefaults = fileSystem.getServerDefaults(path);
  short replication = fileSystem.getDefaultReplication(path);
  EnumSet<CreateFlag> flags = EnumSet.of(CreateFlag.CREATE);
  if (Boolean.getBoolean(HDFS_SYNC_BLOCK)) {
    flags.add(CreateFlag.SYNC_BLOCK);
  }
  return fileSystem.create(path, FsPermission.getDefault()
      .applyUMask(FsPermission.getUMask(conf)), flags, fsDefaults
      .getFileBufferSize(), replication, fsDefaults
      .getBlockSize(), null);
}
 
Example 4
Source Project: indexr   File: SegmentHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static void uploadSegment(StorageSegment segment,
                                 FileSystem fileSystem,
                                 Path path,
                                 Path tableLocation) throws IOException {
    short replica = fileSystem.getDefaultReplication(tableLocation);
    if (replica <= 0) {
        logger.warn("Failed to get replication from {}", tableLocation);
        replica = fileSystem.getDefaultReplication();
    }
    short _replica = replica;

    ByteBufferWriter.PredictSizeOpener writeOpener = size -> {
        long blockSize = getSegmentBlockSize(fileSystem, size);
        FSDataOutputStream outputStream = fileSystem.create(
                path,
                FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fileSystem.getConf())),
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                4096 * 10,
                _replica,
                blockSize,
                null);
        ByteBufferWriter writer = ByteBufferWriter.of(outputStream, outputStream::close);
        writer.setName(path.toString());
        return writer;
    };
    IntegratedSegment.Fd.create(segment, writeOpener, null);
}
 
Example 5
Source Project: indexr   File: SegmentHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static void uploadSegment(StorageSegment segment,
                                 FileSystem fileSystem,
                                 Path path) throws IOException {
    Path parent = path.getParent();
    short replica = fileSystem.getDefaultReplication(parent);
    if (replica <= 0) {
        logger.warn("Failed to get replication from {}", parent);
        replica = fileSystem.getDefaultReplication();
    }
    short _replica = replica;

    ByteBufferWriter.PredictSizeOpener writeOpener = size -> {
        long blockSize = getSegmentBlockSize(fileSystem, size);
        FSDataOutputStream outputStream = fileSystem.create(
                path,
                FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fileSystem.getConf())),
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                4096 * 10,
                _replica,
                blockSize,
                null);
        ByteBufferWriter writer = ByteBufferWriter.of(outputStream, outputStream::close);
        writer.setName(path.toString());
        return writer;
    };
    IntegratedSegment.Fd.create(segment, writeOpener, null);
}
 
Example 6
Source Project: hadoop   File: RetriableFileCopyCommand.java    License: Apache License 2.0 4 votes vote down vote up
private static short getReplicationFactor(
        EnumSet<FileAttribute> fileAttributes,
        FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
  return fileAttributes.contains(FileAttribute.REPLICATION)?
          sourceFile.getReplication() : targetFS.getDefaultReplication(tmpTargetPath);
}
 
Example 7
Source Project: streamx   File: WALFile.java    License: Apache License 2.0 4 votes vote down vote up
Writer(Configuration conf, Option... opts) throws IOException {
  BlockSizeOption blockSizeOption =
      Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption =
      Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption =
      Options.getOption(ReplicationOption.class, opts);

  FileOption fileOption = Options.getOption(FileOption.class, opts);
  AppendIfExistsOption appendIfExistsOption = Options.getOption(
      AppendIfExistsOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);

  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    fs = p.getFileSystem(conf);
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
                     bufferSizeOption.getValue();
    short replication = replicationOption == null ?
                        fs.getDefaultReplication(p) :
                        (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
                     blockSizeOption.getValue();

    if (appendIfExistsOption != null && appendIfExistsOption.getValue()
        && fs.exists(p)) {
      // Read the file and verify header details
      try (WALFile.Reader reader =
               new WALFile.Reader(conf, WALFile.Reader.file(p), new Reader.OnlyHeaderOption())){
        if (reader.getVersion() != VERSION[3]) {
          throw new VersionMismatchException(VERSION[3], reader.getVersion());
        }
        sync = reader.getSync();
      }
      out = fs.append(p, bufferSize);
      this.appendMode = true;
    } else {
      out = fs.create(p, true, bufferSize, replication, blockSize);
    }
  } else {
    out = streamOption.getValue();
  }

  init(conf, out, ownStream);
}
 
Example 8
Source Project: big-c   File: RetriableFileCopyCommand.java    License: Apache License 2.0 4 votes vote down vote up
private static short getReplicationFactor(
        EnumSet<FileAttribute> fileAttributes,
        FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
  return fileAttributes.contains(FileAttribute.REPLICATION)?
          sourceFile.getReplication() : targetFS.getDefaultReplication(tmpTargetPath);
}
 
Example 9
Source Project: hudi   File: FSUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static Short getDefaultReplication(FileSystem fs, Path path) {
  return fs.getDefaultReplication(path);
}
 
Example 10
Source Project: indexr   File: SegmentHelper.java    License: Apache License 2.0 4 votes vote down vote up
public static SegmentFd uploadSegment(StorageSegment segment, FileSystem fileSystem, Path path, boolean openSegment) throws IOException {
    // First upload to a tmp file, then rename to real path after completely transfer all data, to avoid file corruption.

    Path tmpPath = new Path(path.toString() + ".__TMP__");
    SegmentFd fd;
    short replica = fileSystem.getDefaultReplication(path);
    if (replica <= 0) {
        logger.warn("Failed to get replication from {}", path);
        replica = fileSystem.getDefaultReplication();
    }
    short _replica = replica;

    ByteBufferWriter.PredictSizeOpener writeOpener = size -> {
        long blockSize = getSegmentBlockSize(fileSystem, size);
        FSDataOutputStream outputStream = fileSystem.create(
                tmpPath,
                FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(fileSystem.getConf())),
                EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE),
                4096 * 10,
                _replica,
                blockSize,
                null);
        ByteBufferWriter writer = ByteBufferWriter.of(outputStream, outputStream::close);
        writer.setName(tmpPath.toString());
        return writer;
    };
    ByteBufferReader.Opener readerOpener = openSegment ? ByteBufferReader.Opener.create(fileSystem, path) : null;
    fd = IntegratedSegment.Fd.create(segment, writeOpener, readerOpener);

    int times = 0;
    Boolean ok = false;
    while (times < 5) {
        if (fileSystem instanceof DistributedFileSystem) {
            ok = Try.on(() -> ((DistributedFileSystem) fileSystem).rename(tmpPath, path, Options.Rename.OVERWRITE), 1, logger);
        } else {
            ok = Try.on(() -> fileSystem.rename(tmpPath, path), 1, logger);
        }
        if (ok != null && ok) {
            break;
        }
        times++;
        if (fileSystem.exists(path)) {
            Try.on(() -> fileSystem.delete(path), 1, logger, String.format("delete old segment failed: %s", path.toString()));
        }
    }
    if (ok == null || !ok) {
        throw new IOException(String.format("Failed to rename from [%s] to [%s]", tmpPath, path));
    }

    return fd;
}
 
Example 11
Source Project: hbase   File: CommonFSUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static short getDefaultReplication(final FileSystem fs, final Path path) {
  return fs.getDefaultReplication(path);
}
 
Example 12
Source Project: incubator-gobblin   File: CopyableFile.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * @return desired replication for destination file.
 */
public short getReplication(FileSystem targetFs) {
  return getPreserve().preserve(PreserveAttributes.Option.REPLICATION) ?
      getOrigin().getReplication() : targetFs.getDefaultReplication(this.destination);
}
 
Example 13
Source Project: RDFS   File: RaidShell.java    License: Apache License 2.0 4 votes vote down vote up
public void checkFile(String cmd, String[] args, int startIndex)
    throws IOException {
  if (startIndex >= args.length) {
    printUsage(cmd);
    throw new IllegalArgumentException("Insufficient arguments");
  }
  for (int i = startIndex; i < args.length; i++) {
    Path p = new Path(args[i]);
    FileSystem fs = p.getFileSystem(conf);
    // if we got a raid fs, get the underlying fs 
    if (fs instanceof DistributedRaidFileSystem) {
      fs = ((DistributedRaidFileSystem) fs).getFileSystem();
    }
    // We should be able to cast at this point.
    DistributedFileSystem dfs = (DistributedFileSystem) fs;
    RemoteIterator<Path> corruptIt = dfs.listCorruptFileBlocks(p);
    int count = 0;
    while (corruptIt.hasNext()) {
      count++;
      Path corruptFile = corruptIt.next();
      // Result of checking.
      String result = null;
      FileStatus stat = fs.getFileStatus(p);
      if (stat.getReplication() < fs.getDefaultReplication()) {
        RaidInfo raidInfo = getFileRaidInfo(corruptFile);
        if (raidInfo.codec == null) {
          result = "Below default replication but no parity file found";
        } else {
          boolean notRecoverable = isFileCorrupt(dfs, corruptFile);
          if (notRecoverable) {
            result = "Missing too many blocks to be recovered " + 
              "using parity file " + raidInfo.parityPair.getPath();
          } else {
            result = "Has missing blocks but can be read using parity file " +
              raidInfo.parityPair.getPath();
          }
        }
      } else {
        result = "At default replication, not raided";
      }
      out.println("Result of checking " + corruptFile + " : " +
        result);
    }
    out.println("Found " + count + " files with missing blocks");
  }
}
 
Example 14
Source Project: incubator-tajo   File: RCFile.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Constructs a RCFile Writer.
 *
 * @param fs
 *          the file system used
 * @param conf
 *          the configuration file
 * @param name
 *          the file name
 * @param progress a progress meter to update as the file is written
 * @param metadata a string to string map in the file header
 * @throws java.io.IOException
 */
public Writer(FileSystem fs, Configuration conf, Path name,
    Progressable progress, Metadata metadata, CompressionCodec codec) throws IOException {
  this(fs, conf, name, fs.getConf().getInt("io.file.buffer.size", 4096),
      fs.getDefaultReplication(), fs.getDefaultBlockSize(), progress,
      metadata, codec);
}