org.apache.hadoop.util.Options Java Examples

The following examples show how to use org.apache.hadoop.util.Options. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MapFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected synchronized void open(Path dir,
                                 WritableComparator comparator,
                                 Configuration conf, 
                                 SequenceFile.Reader.Option... options
                                 ) throws IOException {
  Path dataFile = new Path(dir, DATA_FILE_NAME);
  Path indexFile = new Path(dir, INDEX_FILE_NAME);

  // open the data
  this.data = createDataFileReader(dataFile, conf, options);
  this.firstPosition = data.getPosition();

  if (comparator == null) {
    Class<? extends WritableComparable> cls;
    cls = data.getKeyClass().asSubclass(WritableComparable.class);
    this.comparator = WritableComparator.get(cls, conf);
  } else {
    this.comparator = comparator;
  }

  // open the index
  SequenceFile.Reader.Option[] indexOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(indexFile));
  this.index = new SequenceFile.Reader(conf, indexOptions);
}
 
Example #2
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
 
Example #3
Source File: SequenceFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
 
Example #4
Source File: SequenceFile.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
 
Example #5
Source File: MapFile.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected synchronized void open(Path dir,
                                 WritableComparator comparator,
                                 Configuration conf, 
                                 SequenceFile.Reader.Option... options
                                 ) throws IOException {
  Path dataFile = new Path(dir, DATA_FILE_NAME);
  Path indexFile = new Path(dir, INDEX_FILE_NAME);

  // open the data
  this.data = createDataFileReader(dataFile, conf, options);
  this.firstPosition = data.getPosition();

  if (comparator == null) {
    Class<? extends WritableComparable> cls;
    cls = data.getKeyClass().asSubclass(WritableComparable.class);
    this.comparator = WritableComparator.get(cls, conf);
  } else {
    this.comparator = comparator;
  }

  // open the index
  SequenceFile.Reader.Option[] indexOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(indexFile));
  this.index = new SequenceFile.Reader(conf, indexOptions);
}
 
Example #6
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
 
Example #7
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 5 votes vote down vote up
public Reader(Configuration conf, Option... opts) throws IOException {
  // Look up the options, these are null if not set
  FileOption fileOpt = Options.getOption(FileOption.class, opts);
  InputStreamOption streamOpt = 
    Options.getOption(InputStreamOption.class, opts);
  StartOption startOpt = Options.getOption(StartOption.class, opts);
  LengthOption lenOpt = Options.getOption(LengthOption.class, opts);
  BufferSizeOption bufOpt = Options.getOption(BufferSizeOption.class,opts);
  OnlyHeaderOption headerOnly = 
    Options.getOption(OnlyHeaderOption.class, opts);
  // check for consistency
  if ((fileOpt == null) == (streamOpt == null)) {
    throw new 
      IllegalArgumentException("File or stream option must be specified");
  }
  if (fileOpt == null && bufOpt != null) {
    throw new IllegalArgumentException("buffer size can only be set when" +
                                       " a file is specified.");
  }
  // figure out the real values
  Path filename = null;
  FSDataInputStream file;
  final long len;
  if (fileOpt != null) {
    filename = fileOpt.getValue();
    FileSystem fs = filename.getFileSystem(conf);
    int bufSize = bufOpt == null ? getBufferSize(conf): bufOpt.getValue();
    len = null == lenOpt
      ? fs.getFileStatus(filename).getLen()
      : lenOpt.getValue();
    file = openFile(fs, filename, bufSize, len);
  } else {
    len = null == lenOpt ? Long.MAX_VALUE : lenOpt.getValue();
    file = streamOpt.getValue();
  }
  long start = startOpt == null ? 0 : startOpt.getValue();
  // really set up
  initialize(filename, file, start, len, conf, headerOnly != null);
}
 
Example #8
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 5 votes vote down vote up
public Reader(Configuration conf, Option... opts) throws IOException {
  // Look up the options, these are null if not set
  FileOption fileOpt = Options.getOption(FileOption.class, opts);
  InputStreamOption streamOpt = 
    Options.getOption(InputStreamOption.class, opts);
  StartOption startOpt = Options.getOption(StartOption.class, opts);
  LengthOption lenOpt = Options.getOption(LengthOption.class, opts);
  BufferSizeOption bufOpt = Options.getOption(BufferSizeOption.class,opts);
  OnlyHeaderOption headerOnly = 
    Options.getOption(OnlyHeaderOption.class, opts);
  // check for consistency
  if ((fileOpt == null) == (streamOpt == null)) {
    throw new 
      IllegalArgumentException("File or stream option must be specified");
  }
  if (fileOpt == null && bufOpt != null) {
    throw new IllegalArgumentException("buffer size can only be set when" +
                                       " a file is specified.");
  }
  // figure out the real values
  Path filename = null;
  FSDataInputStream file;
  final long len;
  if (fileOpt != null) {
    filename = fileOpt.getValue();
    FileSystem fs = filename.getFileSystem(conf);
    int bufSize = bufOpt == null ? getBufferSize(conf): bufOpt.getValue();
    len = null == lenOpt
      ? fs.getFileStatus(filename).getLen()
      : lenOpt.getValue();
    file = openFile(fs, filename, bufSize, len);
  } else {
    len = null == lenOpt ? Long.MAX_VALUE : lenOpt.getValue();
    file = streamOpt.getValue();
  }
  long start = startOpt == null ? 0 : startOpt.getValue();
  // really set up
  initialize(filename, file, start, len, conf, headerOnly != null);
}
 
Example #9
Source File: SequenceFile.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Reader(Configuration conf, Option... opts) throws IOException {
  // Look up the options, these are null if not set
  FileOption fileOpt = Options.getOption(FileOption.class, opts);
  InputStreamOption streamOpt = 
    Options.getOption(InputStreamOption.class, opts);
  StartOption startOpt = Options.getOption(StartOption.class, opts);
  LengthOption lenOpt = Options.getOption(LengthOption.class, opts);
  BufferSizeOption bufOpt = Options.getOption(BufferSizeOption.class,opts);
  OnlyHeaderOption headerOnly = 
    Options.getOption(OnlyHeaderOption.class, opts);
  // check for consistency
  if ((fileOpt == null) == (streamOpt == null)) {
    throw new 
      IllegalArgumentException("File or stream option must be specified");
  }
  if (fileOpt == null && bufOpt != null) {
    throw new IllegalArgumentException("buffer size can only be set when" +
                                       " a file is specified.");
  }
  // figure out the real values
  Path filename = null;
  FSDataInputStream file;
  final long len;
  if (fileOpt != null) {
    filename = fileOpt.getValue();
    FileSystem fs = filename.getFileSystem(conf);
    int bufSize = bufOpt == null ? getBufferSize(conf): bufOpt.getValue();
    len = null == lenOpt
      ? fs.getFileStatus(filename).getLen()
      : lenOpt.getValue();
    file = openFile(fs, filename, bufSize, len);
  } else {
    len = null == lenOpt ? Long.MAX_VALUE : lenOpt.getValue();
    file = streamOpt.getValue();
  }
  long start = startOpt == null ? 0 : startOpt.getValue();
  // really set up
  initialize(filename, file, start, len, conf, headerOnly != null);
}
 
Example #10
Source File: MapFile.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Override this method to specialize the type of
 * {@link SequenceFile.Reader} returned.
 */
protected SequenceFile.Reader 
  createDataFileReader(Path dataFile, Configuration conf,
                       SequenceFile.Reader.Option... options
                       ) throws IOException {
  SequenceFile.Reader.Option[] newOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(dataFile));
  return new SequenceFile.Reader(conf, newOptions);
}
 
Example #11
Source File: MapFile.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Reader(Path dir, Configuration conf,
              SequenceFile.Reader.Option... opts) throws IOException {
  ComparatorOption comparatorOption = 
    Options.getOption(ComparatorOption.class, opts);
  WritableComparator comparator =
    comparatorOption == null ? null : comparatorOption.getValue();
  INDEX_SKIP = conf.getInt("io.map.index.skip", 0);
  open(dir, comparator, conf, opts);
}
 
Example #12
Source File: WALFile.java    From streamx with Apache License 2.0 5 votes vote down vote up
public Reader(Configuration conf, Option... opts) throws IOException {
  // Look up the options, these are null if not set
  FileOption fileOpt = Options.getOption(FileOption.class, opts);
  InputStreamOption streamOpt =
      Options.getOption(InputStreamOption.class, opts);
  StartOption startOpt = Options.getOption(StartOption.class, opts);
  LengthOption lenOpt = Options.getOption(LengthOption.class, opts);
  BufferSizeOption bufOpt = Options.getOption(BufferSizeOption.class, opts);
  OnlyHeaderOption headerOnly =
      Options.getOption(OnlyHeaderOption.class, opts);
  // check for consistency
  if ((fileOpt == null) == (streamOpt == null)) {
    throw new
        IllegalArgumentException("File or stream option must be specified");
  }
  if (fileOpt == null && bufOpt != null) {
    throw new IllegalArgumentException("buffer size can only be set when" +
                                       " a file is specified.");
  }
  // figure out the real values
  Path filename = null;
  FSDataInputStream file;
  final long len;
  if (fileOpt != null) {
    filename = fileOpt.getValue();
    FileSystem fs = filename.getFileSystem(conf);
    int bufSize = bufOpt == null ? getBufferSize(conf) : bufOpt.getValue();
    len = null == lenOpt
          ? fs.getFileStatus(filename).getLen()
          : lenOpt.getValue();
    file = openFile(fs, filename, bufSize, len);
  } else {
    len = null == lenOpt ? Long.MAX_VALUE : lenOpt.getValue();
    file = streamOpt.getValue();
  }
  long start = startOpt == null ? 0 : startOpt.getValue();
  // really set up
  initialize(filename, file, start, len, conf, headerOnly != null);
}
 
Example #13
Source File: SequenceFile.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public Reader(Configuration conf, Option... opts) throws IOException {
  // Look up the options, these are null if not set
  FileOption fileOpt = Options.getOption(FileOption.class, opts);
  InputStreamOption streamOpt = 
    Options.getOption(InputStreamOption.class, opts);
  StartOption startOpt = Options.getOption(StartOption.class, opts);
  LengthOption lenOpt = Options.getOption(LengthOption.class, opts);
  BufferSizeOption bufOpt = Options.getOption(BufferSizeOption.class,opts);
  OnlyHeaderOption headerOnly = 
    Options.getOption(OnlyHeaderOption.class, opts);
  // check for consistency
  if ((fileOpt == null) == (streamOpt == null)) {
    throw new 
      IllegalArgumentException("File or stream option must be specified");
  }
  if (fileOpt == null && bufOpt != null) {
    throw new IllegalArgumentException("buffer size can only be set when" +
                                       " a file is specified.");
  }
  // figure out the real values
  Path filename = null;
  FSDataInputStream file;
  final long len;
  if (fileOpt != null) {
    filename = fileOpt.getValue();
    FileSystem fs = filename.getFileSystem(conf);
    int bufSize = bufOpt == null ? getBufferSize(conf): bufOpt.getValue();
    len = null == lenOpt
      ? fs.getFileStatus(filename).getLen()
      : lenOpt.getValue();
    file = openFile(fs, filename, bufSize, len);
  } else {
    len = null == lenOpt ? Long.MAX_VALUE : lenOpt.getValue();
    file = streamOpt.getValue();
  }
  long start = startOpt == null ? 0 : startOpt.getValue();
  // really set up
  initialize(filename, file, start, len, conf, headerOnly != null);
}
 
Example #14
Source File: MapFile.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Override this method to specialize the type of
 * {@link SequenceFile.Reader} returned.
 */
protected SequenceFile.Reader 
  createDataFileReader(Path dataFile, Configuration conf,
                       SequenceFile.Reader.Option... options
                       ) throws IOException {
  SequenceFile.Reader.Option[] newOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(dataFile));
  return new SequenceFile.Reader(conf, newOptions);
}
 
Example #15
Source File: MapFile.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public Reader(Path dir, Configuration conf,
              SequenceFile.Reader.Option... opts) throws IOException {
  ComparatorOption comparatorOption = 
    Options.getOption(ComparatorOption.class, opts);
  WritableComparator comparator =
    comparatorOption == null ? null : comparatorOption.getValue();
  INDEX_SKIP = conf.getInt("io.map.index.skip", 0);
  open(dir, comparator, conf, opts);
}
 
Example #16
Source File: MapFile.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public Writer(Configuration conf, 
              Path dirName,
              SequenceFile.Writer.Option... opts
              ) throws IOException {
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ComparatorOption comparatorOption =
    Options.getOption(ComparatorOption.class, opts);
  if ((keyClassOption == null) == (comparatorOption == null)) {
    throw new IllegalArgumentException("key class or comparator option "
                                       + "must be set");
  }
  this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval);

  Class<? extends WritableComparable> keyClass;
  if (keyClassOption == null) {
    this.comparator = comparatorOption.getValue();
    keyClass = comparator.getKeyClass();
  } else {
    keyClass= 
      (Class<? extends WritableComparable>) keyClassOption.getValue();
    this.comparator = WritableComparator.get(keyClass, conf);
  }
  this.lastKey = comparator.newKey();
  FileSystem fs = dirName.getFileSystem(conf);

  if (!fs.mkdirs(dirName)) {
    throw new IOException("Mkdirs failed to create directory " + dirName);
  }
  Path dataFile = new Path(dirName, DATA_FILE_NAME);
  Path indexFile = new Path(dirName, INDEX_FILE_NAME);

  SequenceFile.Writer.Option[] dataOptions =
    Options.prependOptions(opts, 
                           SequenceFile.Writer.file(dataFile),
                           SequenceFile.Writer.keyClass(keyClass));
  this.data = SequenceFile.createWriter(conf, dataOptions);

  SequenceFile.Writer.Option[] indexOptions =
    Options.prependOptions(opts, SequenceFile.Writer.file(indexFile),
        SequenceFile.Writer.keyClass(keyClass),
        SequenceFile.Writer.valueClass(LongWritable.class),
        SequenceFile.Writer.compression(CompressionType.BLOCK));
  this.index = SequenceFile.createWriter(conf, indexOptions);      
}
 
Example #17
Source File: MapFile.java    From big-c with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
public Writer(Configuration conf, 
              Path dirName,
              SequenceFile.Writer.Option... opts
              ) throws IOException {
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ComparatorOption comparatorOption =
    Options.getOption(ComparatorOption.class, opts);
  if ((keyClassOption == null) == (comparatorOption == null)) {
    throw new IllegalArgumentException("key class or comparator option "
                                       + "must be set");
  }
  this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval);

  Class<? extends WritableComparable> keyClass;
  if (keyClassOption == null) {
    this.comparator = comparatorOption.getValue();
    keyClass = comparator.getKeyClass();
  } else {
    keyClass= 
      (Class<? extends WritableComparable>) keyClassOption.getValue();
    this.comparator = WritableComparator.get(keyClass, conf);
  }
  this.lastKey = comparator.newKey();
  FileSystem fs = dirName.getFileSystem(conf);

  if (!fs.mkdirs(dirName)) {
    throw new IOException("Mkdirs failed to create directory " + dirName);
  }
  Path dataFile = new Path(dirName, DATA_FILE_NAME);
  Path indexFile = new Path(dirName, INDEX_FILE_NAME);

  SequenceFile.Writer.Option[] dataOptions =
    Options.prependOptions(opts, 
                           SequenceFile.Writer.file(dataFile),
                           SequenceFile.Writer.keyClass(keyClass));
  this.data = SequenceFile.createWriter(conf, dataOptions);

  SequenceFile.Writer.Option[] indexOptions =
    Options.prependOptions(opts, SequenceFile.Writer.file(indexFile),
        SequenceFile.Writer.keyClass(keyClass),
        SequenceFile.Writer.valueClass(LongWritable.class),
        SequenceFile.Writer.compression(CompressionType.BLOCK));
  this.index = SequenceFile.createWriter(conf, indexOptions);      
}
 
Example #18
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 4 votes vote down vote up
/**
 * Construct a uncompressed writer from a set of options.
 * @param conf the configuration to use
 * @param opts the options used when creating the writer
 * @throws IOException if it fails
 */
Writer(Configuration conf, 
       Option... opts) throws IOException {
  BlockSizeOption blockSizeOption = 
    Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption = 
    Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption = 
    Options.getOption(ReplicationOption.class, opts);
  ProgressableOption progressOption = 
    Options.getOption(ProgressableOption.class, opts);
  FileOption fileOption = Options.getOption(FileOption.class, opts);
  FileSystemOption fsOption = Options.getOption(FileSystemOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ValueClassOption valueClassOption = 
    Options.getOption(ValueClassOption.class, opts);
  MetadataOption metadataOption = 
    Options.getOption(MetadataOption.class, opts);
  CompressionOption compressionTypeOption =
    Options.getOption(CompressionOption.class, opts);
  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null ||
                             progressOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    if (fsOption != null) {
      fs = fsOption.getValue();
    } else {
      fs = p.getFileSystem(conf);
    }
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
      bufferSizeOption.getValue();
    short replication = replicationOption == null ? 
      fs.getDefaultReplication(p) :
      (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
      blockSizeOption.getValue();
    Progressable progress = progressOption == null ? null :
      progressOption.getValue();
    out = fs.create(p, true, bufferSize, replication, blockSize, progress);
  } else {
    out = streamOption.getValue();
  }
  Class<?> keyClass = keyClassOption == null ?
      Object.class : keyClassOption.getValue();
  Class<?> valueClass = valueClassOption == null ?
      Object.class : valueClassOption.getValue();
  Metadata metadata = metadataOption == null ?
      new Metadata() : metadataOption.getValue();
  this.compress = compressionTypeOption.getValue();
  final CompressionCodec codec = compressionTypeOption.getCodec();
  if (codec != null &&
      (codec instanceof GzipCodec) &&
      !NativeCodeLoader.isNativeCodeLoaded() &&
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop " +
                                       "code!");
  }
  init(conf, out, ownStream, keyClass, valueClass, codec, metadata);
}
 
Example #19
Source File: WALFile.java    From streamx with Apache License 2.0 4 votes vote down vote up
Writer(Configuration conf, Option... opts) throws IOException {
  BlockSizeOption blockSizeOption =
      Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption =
      Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption =
      Options.getOption(ReplicationOption.class, opts);

  FileOption fileOption = Options.getOption(FileOption.class, opts);
  AppendIfExistsOption appendIfExistsOption = Options.getOption(
      AppendIfExistsOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);

  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    fs = p.getFileSystem(conf);
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
                     bufferSizeOption.getValue();
    short replication = replicationOption == null ?
                        fs.getDefaultReplication(p) :
                        (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
                     blockSizeOption.getValue();

    if (appendIfExistsOption != null && appendIfExistsOption.getValue()
        && fs.exists(p)) {
      // Read the file and verify header details
      try (WALFile.Reader reader =
               new WALFile.Reader(conf, WALFile.Reader.file(p), new Reader.OnlyHeaderOption())){
        if (reader.getVersion() != VERSION[3]) {
          throw new VersionMismatchException(VERSION[3], reader.getVersion());
        }
        sync = reader.getSync();
      }
      out = fs.append(p, bufferSize);
      this.appendMode = true;
    } else {
      out = fs.create(p, true, bufferSize, replication, blockSize);
    }
  } else {
    out = streamOption.getValue();
  }

  init(conf, out, ownStream);
}
 
Example #20
Source File: SequenceFile.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Construct a uncompressed writer from a set of options.
 * @param conf the configuration to use
 * @param options the options used when creating the writer
 * @throws IOException if it fails
 */
Writer(Configuration conf, 
       Option... opts) throws IOException {
  BlockSizeOption blockSizeOption = 
    Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption = 
    Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption = 
    Options.getOption(ReplicationOption.class, opts);
  ProgressableOption progressOption = 
    Options.getOption(ProgressableOption.class, opts);
  FileOption fileOption = Options.getOption(FileOption.class, opts);
  FileSystemOption fsOption = Options.getOption(FileSystemOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ValueClassOption valueClassOption = 
    Options.getOption(ValueClassOption.class, opts);
  MetadataOption metadataOption = 
    Options.getOption(MetadataOption.class, opts);
  CompressionOption compressionTypeOption =
    Options.getOption(CompressionOption.class, opts);
  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null ||
                             progressOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    if (fsOption != null) {
      fs = fsOption.getValue();
    } else {
      fs = p.getFileSystem(conf);
    }
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
      bufferSizeOption.getValue();
    short replication = replicationOption == null ? 
      fs.getDefaultReplication(p) :
      (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
      blockSizeOption.getValue();
    Progressable progress = progressOption == null ? null :
      progressOption.getValue();
    out = fs.create(p, true, bufferSize, replication, blockSize, progress);
  } else {
    out = streamOption.getValue();
  }
  Class<?> keyClass = keyClassOption == null ?
      Object.class : keyClassOption.getValue();
  Class<?> valueClass = valueClassOption == null ?
      Object.class : valueClassOption.getValue();
  Metadata metadata = metadataOption == null ?
      new Metadata() : metadataOption.getValue();
  this.compress = compressionTypeOption.getValue();
  final CompressionCodec codec = compressionTypeOption.getCodec();
  if (codec != null &&
      (codec instanceof GzipCodec) &&
      !NativeCodeLoader.isNativeCodeLoaded() &&
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop " +
                                       "code!");
  }
  init(conf, out, ownStream, keyClass, valueClass, codec, metadata);
}
 
Example #21
Source File: SequenceFile.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Construct a uncompressed writer from a set of options.
 * @param conf the configuration to use
 * @param options the options used when creating the writer
 * @throws IOException if it fails
 */
Writer(Configuration conf, 
       Option... opts) throws IOException {
  BlockSizeOption blockSizeOption = 
    Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption = 
    Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption = 
    Options.getOption(ReplicationOption.class, opts);
  ProgressableOption progressOption = 
    Options.getOption(ProgressableOption.class, opts);
  FileOption fileOption = Options.getOption(FileOption.class, opts);
  FileSystemOption fsOption = Options.getOption(FileSystemOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ValueClassOption valueClassOption = 
    Options.getOption(ValueClassOption.class, opts);
  MetadataOption metadataOption = 
    Options.getOption(MetadataOption.class, opts);
  CompressionOption compressionTypeOption =
    Options.getOption(CompressionOption.class, opts);
  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null ||
                             progressOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    if (fsOption != null) {
      fs = fsOption.getValue();
    } else {
      fs = p.getFileSystem(conf);
    }
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
      bufferSizeOption.getValue();
    short replication = replicationOption == null ? 
      fs.getDefaultReplication(p) :
      (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
      blockSizeOption.getValue();
    Progressable progress = progressOption == null ? null :
      progressOption.getValue();
    out = fs.create(p, true, bufferSize, replication, blockSize, progress);
  } else {
    out = streamOption.getValue();
  }
  Class<?> keyClass = keyClassOption == null ?
      Object.class : keyClassOption.getValue();
  Class<?> valueClass = valueClassOption == null ?
      Object.class : valueClassOption.getValue();
  Metadata metadata = metadataOption == null ?
      new Metadata() : metadataOption.getValue();
  this.compress = compressionTypeOption.getValue();
  final CompressionCodec codec = compressionTypeOption.getCodec();
  if (codec != null &&
      (codec instanceof GzipCodec) &&
      !NativeCodeLoader.isNativeCodeLoaded() &&
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop " +
                                       "code!");
  }
  init(conf, out, ownStream, keyClass, valueClass, codec, metadata);
}
 
Example #22
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 4 votes vote down vote up
/**
 * Construct a uncompressed writer from a set of options.
 * @param conf the configuration to use
 * @param opts the options used when creating the writer
 * @throws IOException if it fails
 */
Writer(Configuration conf, 
       Option... opts) throws IOException {
  BlockSizeOption blockSizeOption = 
    Options.getOption(BlockSizeOption.class, opts);
  BufferSizeOption bufferSizeOption = 
    Options.getOption(BufferSizeOption.class, opts);
  ReplicationOption replicationOption = 
    Options.getOption(ReplicationOption.class, opts);
  ProgressableOption progressOption = 
    Options.getOption(ProgressableOption.class, opts);
  FileOption fileOption = Options.getOption(FileOption.class, opts);
  FileSystemOption fsOption = Options.getOption(FileSystemOption.class, opts);
  StreamOption streamOption = Options.getOption(StreamOption.class, opts);
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ValueClassOption valueClassOption = 
    Options.getOption(ValueClassOption.class, opts);
  MetadataOption metadataOption = 
    Options.getOption(MetadataOption.class, opts);
  CompressionOption compressionTypeOption =
    Options.getOption(CompressionOption.class, opts);
  // check consistency of options
  if ((fileOption == null) == (streamOption == null)) {
    throw new IllegalArgumentException("file or stream must be specified");
  }
  if (fileOption == null && (blockSizeOption != null ||
                             bufferSizeOption != null ||
                             replicationOption != null ||
                             progressOption != null)) {
    throw new IllegalArgumentException("file modifier options not " +
                                       "compatible with stream");
  }

  FSDataOutputStream out;
  boolean ownStream = fileOption != null;
  if (ownStream) {
    Path p = fileOption.getValue();
    FileSystem fs;
    if (fsOption != null) {
      fs = fsOption.getValue();
    } else {
      fs = p.getFileSystem(conf);
    }
    int bufferSize = bufferSizeOption == null ? getBufferSize(conf) :
      bufferSizeOption.getValue();
    short replication = replicationOption == null ? 
      fs.getDefaultReplication(p) :
      (short) replicationOption.getValue();
    long blockSize = blockSizeOption == null ? fs.getDefaultBlockSize(p) :
      blockSizeOption.getValue();
    Progressable progress = progressOption == null ? null :
      progressOption.getValue();
    out = fs.create(p, true, bufferSize, replication, blockSize, progress);
  } else {
    out = streamOption.getValue();
  }
  Class<?> keyClass = keyClassOption == null ?
      Object.class : keyClassOption.getValue();
  Class<?> valueClass = valueClassOption == null ?
      Object.class : valueClassOption.getValue();
  Metadata metadata = metadataOption == null ?
      new Metadata() : metadataOption.getValue();
  this.compress = compressionTypeOption.getValue();
  final CompressionCodec codec = compressionTypeOption.getCodec();
  if (codec != null &&
      (codec instanceof GzipCodec) &&
      !NativeCodeLoader.isNativeCodeLoaded() &&
      !ZlibFactory.isNativeZlibLoaded(conf)) {
    throw new IllegalArgumentException("SequenceFile doesn't work with " +
                                       "GzipCodec without native-hadoop " +
                                       "code!");
  }
  init(conf, out, ownStream, keyClass, valueClass, codec, metadata);
}