Java Code Examples for org.apache.hadoop.util.Options#prependOptions()

The following examples show how to use org.apache.hadoop.util.Options#prependOptions() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MapFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
protected synchronized void open(Path dir,
                                 WritableComparator comparator,
                                 Configuration conf, 
                                 SequenceFile.Reader.Option... options
                                 ) throws IOException {
  Path dataFile = new Path(dir, DATA_FILE_NAME);
  Path indexFile = new Path(dir, INDEX_FILE_NAME);

  // open the data
  this.data = createDataFileReader(dataFile, conf, options);
  this.firstPosition = data.getPosition();

  if (comparator == null) {
    Class<? extends WritableComparable> cls;
    cls = data.getKeyClass().asSubclass(WritableComparable.class);
    this.comparator = WritableComparator.get(cls, conf);
  } else {
    this.comparator = comparator;
  }

  // open the index
  SequenceFile.Reader.Option[] indexOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(indexFile));
  this.index = new SequenceFile.Reader(conf, indexOptions);
}
Example 2
Source File: SequenceFile.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
Example 3
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up 
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
Example 4
Source File: MapFile.java    From big-c with Apache License 2.0 6 votes vote down vote up 
protected synchronized void open(Path dir,
                                 WritableComparator comparator,
                                 Configuration conf, 
                                 SequenceFile.Reader.Option... options
                                 ) throws IOException {
  Path dataFile = new Path(dir, DATA_FILE_NAME);
  Path indexFile = new Path(dir, INDEX_FILE_NAME);

  // open the data
  this.data = createDataFileReader(dataFile, conf, options);
  this.firstPosition = data.getPosition();

  if (comparator == null) {
    Class<? extends WritableComparable> cls;
    cls = data.getKeyClass().asSubclass(WritableComparable.class);
    this.comparator = WritableComparator.get(cls, conf);
  } else {
    this.comparator = comparator;
  }

  // open the index
  SequenceFile.Reader.Option[] indexOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(indexFile));
  this.index = new SequenceFile.Reader(conf, indexOptions);
}
Example 5
Source File: SequenceFile.java    From big-c with Apache License 2.0 6 votes vote down vote up 
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
Example 6
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up 
/**
 * Create a new Writer with the given options.
 * @param conf the configuration to use
 * @param opts the options to create the file with
 * @return a new Writer
 * @throws IOException
 */
public static Writer createWriter(Configuration conf, Writer.Option... opts
                                  ) throws IOException {
  Writer.CompressionOption compressionOption = 
    Options.getOption(Writer.CompressionOption.class, opts);
  CompressionType kind;
  if (compressionOption != null) {
    kind = compressionOption.getValue();
  } else {
    kind = getDefaultCompressionType(conf);
    opts = Options.prependOptions(opts, Writer.compression(kind));
  }
  switch (kind) {
    default:
    case NONE:
      return new Writer(conf, opts);
    case RECORD:
      return new RecordCompressWriter(conf, opts);
    case BLOCK:
      return new BlockCompressWriter(conf, opts);
  }
}
Example 7
Source File: MapFile.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Override this method to specialize the type of
 * {@link SequenceFile.Reader} returned.
 */
protected SequenceFile.Reader 
  createDataFileReader(Path dataFile, Configuration conf,
                       SequenceFile.Reader.Option... options
                       ) throws IOException {
  SequenceFile.Reader.Option[] newOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(dataFile));
  return new SequenceFile.Reader(conf, newOptions);
}
Example 8
Source File: MapFile.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Override this method to specialize the type of
 * {@link SequenceFile.Reader} returned.
 */
protected SequenceFile.Reader 
  createDataFileReader(Path dataFile, Configuration conf,
                       SequenceFile.Reader.Option... options
                       ) throws IOException {
  SequenceFile.Reader.Option[] newOptions =
    Options.prependOptions(options, SequenceFile.Reader.file(dataFile));
  return new SequenceFile.Reader(conf, newOptions);
}
Example 9
Source File: MapFile.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
public Writer(Configuration conf, 
              Path dirName,
              SequenceFile.Writer.Option... opts
              ) throws IOException {
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ComparatorOption comparatorOption =
    Options.getOption(ComparatorOption.class, opts);
  if ((keyClassOption == null) == (comparatorOption == null)) {
    throw new IllegalArgumentException("key class or comparator option "
                                       + "must be set");
  }
  this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval);

  Class<? extends WritableComparable> keyClass;
  if (keyClassOption == null) {
    this.comparator = comparatorOption.getValue();
    keyClass = comparator.getKeyClass();
  } else {
    keyClass= 
      (Class<? extends WritableComparable>) keyClassOption.getValue();
    this.comparator = WritableComparator.get(keyClass, conf);
  }
  this.lastKey = comparator.newKey();
  FileSystem fs = dirName.getFileSystem(conf);

  if (!fs.mkdirs(dirName)) {
    throw new IOException("Mkdirs failed to create directory " + dirName);
  }
  Path dataFile = new Path(dirName, DATA_FILE_NAME);
  Path indexFile = new Path(dirName, INDEX_FILE_NAME);

  SequenceFile.Writer.Option[] dataOptions =
    Options.prependOptions(opts, 
                           SequenceFile.Writer.file(dataFile),
                           SequenceFile.Writer.keyClass(keyClass));
  this.data = SequenceFile.createWriter(conf, dataOptions);

  SequenceFile.Writer.Option[] indexOptions =
    Options.prependOptions(opts, SequenceFile.Writer.file(indexFile),
        SequenceFile.Writer.keyClass(keyClass),
        SequenceFile.Writer.valueClass(LongWritable.class),
        SequenceFile.Writer.compression(CompressionType.BLOCK));
  this.index = SequenceFile.createWriter(conf, indexOptions);      
}
Example 10
Source File: MapFile.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
public Writer(Configuration conf, 
              Path dirName,
              SequenceFile.Writer.Option... opts
              ) throws IOException {
  KeyClassOption keyClassOption = 
    Options.getOption(KeyClassOption.class, opts);
  ComparatorOption comparatorOption =
    Options.getOption(ComparatorOption.class, opts);
  if ((keyClassOption == null) == (comparatorOption == null)) {
    throw new IllegalArgumentException("key class or comparator option "
                                       + "must be set");
  }
  this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval);

  Class<? extends WritableComparable> keyClass;
  if (keyClassOption == null) {
    this.comparator = comparatorOption.getValue();
    keyClass = comparator.getKeyClass();
  } else {
    keyClass= 
      (Class<? extends WritableComparable>) keyClassOption.getValue();
    this.comparator = WritableComparator.get(keyClass, conf);
  }
  this.lastKey = comparator.newKey();
  FileSystem fs = dirName.getFileSystem(conf);

  if (!fs.mkdirs(dirName)) {
    throw new IOException("Mkdirs failed to create directory " + dirName);
  }
  Path dataFile = new Path(dirName, DATA_FILE_NAME);
  Path indexFile = new Path(dirName, INDEX_FILE_NAME);

  SequenceFile.Writer.Option[] dataOptions =
    Options.prependOptions(opts, 
                           SequenceFile.Writer.file(dataFile),
                           SequenceFile.Writer.keyClass(keyClass));
  this.data = SequenceFile.createWriter(conf, dataOptions);

  SequenceFile.Writer.Option[] indexOptions =
    Options.prependOptions(opts, SequenceFile.Writer.file(indexFile),
        SequenceFile.Writer.keyClass(keyClass),
        SequenceFile.Writer.valueClass(LongWritable.class),
        SequenceFile.Writer.compression(CompressionType.BLOCK));
  this.index = SequenceFile.createWriter(conf, indexOptions);      
}