Java Code Examples for org.apache.hadoop.util.Options#prependOptions()
The following examples show how to use
org.apache.hadoop.util.Options#prependOptions() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MapFile.java From hadoop with Apache License 2.0 | 6 votes |
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
Example 2
Source File: SequenceFile.java From hadoop with Apache License 2.0 | 6 votes |
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
Example 3
Source File: SequenceFile.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
Example 4
Source File: MapFile.java From big-c with Apache License 2.0 | 6 votes |
protected synchronized void open(Path dir, WritableComparator comparator, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { Path dataFile = new Path(dir, DATA_FILE_NAME); Path indexFile = new Path(dir, INDEX_FILE_NAME); // open the data this.data = createDataFileReader(dataFile, conf, options); this.firstPosition = data.getPosition(); if (comparator == null) { Class<? extends WritableComparable> cls; cls = data.getKeyClass().asSubclass(WritableComparable.class); this.comparator = WritableComparator.get(cls, conf); } else { this.comparator = comparator; } // open the index SequenceFile.Reader.Option[] indexOptions = Options.prependOptions(options, SequenceFile.Reader.file(indexFile)); this.index = new SequenceFile.Reader(conf, indexOptions); }
Example 5
Source File: SequenceFile.java From big-c with Apache License 2.0 | 6 votes |
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
Example 6
Source File: SequenceFile.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
/** * Create a new Writer with the given options. * @param conf the configuration to use * @param opts the options to create the file with * @return a new Writer * @throws IOException */ public static Writer createWriter(Configuration conf, Writer.Option... opts ) throws IOException { Writer.CompressionOption compressionOption = Options.getOption(Writer.CompressionOption.class, opts); CompressionType kind; if (compressionOption != null) { kind = compressionOption.getValue(); } else { kind = getDefaultCompressionType(conf); opts = Options.prependOptions(opts, Writer.compression(kind)); } switch (kind) { default: case NONE: return new Writer(conf, opts); case RECORD: return new RecordCompressWriter(conf, opts); case BLOCK: return new BlockCompressWriter(conf, opts); } }
Example 7
Source File: MapFile.java From hadoop with Apache License 2.0 | 5 votes |
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
Example 8
Source File: MapFile.java From big-c with Apache License 2.0 | 5 votes |
/** * Override this method to specialize the type of * {@link SequenceFile.Reader} returned. */ protected SequenceFile.Reader createDataFileReader(Path dataFile, Configuration conf, SequenceFile.Reader.Option... options ) throws IOException { SequenceFile.Reader.Option[] newOptions = Options.prependOptions(options, SequenceFile.Reader.file(dataFile)); return new SequenceFile.Reader(conf, newOptions); }
Example 9
Source File: MapFile.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") public Writer(Configuration conf, Path dirName, SequenceFile.Writer.Option... opts ) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " + "must be set"); } this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval); Class<? extends WritableComparable> keyClass; if (keyClassOption == null) { this.comparator = comparatorOption.getValue(); keyClass = comparator.getKeyClass(); } else { keyClass= (Class<? extends WritableComparable>) keyClassOption.getValue(); this.comparator = WritableComparator.get(keyClass, conf); } this.lastKey = comparator.newKey(); FileSystem fs = dirName.getFileSystem(conf); if (!fs.mkdirs(dirName)) { throw new IOException("Mkdirs failed to create directory " + dirName); } Path dataFile = new Path(dirName, DATA_FILE_NAME); Path indexFile = new Path(dirName, INDEX_FILE_NAME); SequenceFile.Writer.Option[] dataOptions = Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); this.data = SequenceFile.createWriter(conf, dataOptions); SequenceFile.Writer.Option[] indexOptions = Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class), SequenceFile.Writer.compression(CompressionType.BLOCK)); this.index = SequenceFile.createWriter(conf, indexOptions); }
Example 10
Source File: MapFile.java From big-c with Apache License 2.0 | 4 votes |
@SuppressWarnings("unchecked") public Writer(Configuration conf, Path dirName, SequenceFile.Writer.Option... opts ) throws IOException { KeyClassOption keyClassOption = Options.getOption(KeyClassOption.class, opts); ComparatorOption comparatorOption = Options.getOption(ComparatorOption.class, opts); if ((keyClassOption == null) == (comparatorOption == null)) { throw new IllegalArgumentException("key class or comparator option " + "must be set"); } this.indexInterval = conf.getInt(INDEX_INTERVAL, this.indexInterval); Class<? extends WritableComparable> keyClass; if (keyClassOption == null) { this.comparator = comparatorOption.getValue(); keyClass = comparator.getKeyClass(); } else { keyClass= (Class<? extends WritableComparable>) keyClassOption.getValue(); this.comparator = WritableComparator.get(keyClass, conf); } this.lastKey = comparator.newKey(); FileSystem fs = dirName.getFileSystem(conf); if (!fs.mkdirs(dirName)) { throw new IOException("Mkdirs failed to create directory " + dirName); } Path dataFile = new Path(dirName, DATA_FILE_NAME); Path indexFile = new Path(dirName, INDEX_FILE_NAME); SequenceFile.Writer.Option[] dataOptions = Options.prependOptions(opts, SequenceFile.Writer.file(dataFile), SequenceFile.Writer.keyClass(keyClass)); this.data = SequenceFile.createWriter(conf, dataOptions); SequenceFile.Writer.Option[] indexOptions = Options.prependOptions(opts, SequenceFile.Writer.file(indexFile), SequenceFile.Writer.keyClass(keyClass), SequenceFile.Writer.valueClass(LongWritable.class), SequenceFile.Writer.compression(CompressionType.BLOCK)); this.index = SequenceFile.createWriter(conf, indexOptions); }