Java Code Examples for org.apache.hadoop.io.SequenceFile.CompressionType#NONE

The following examples show how to use org.apache.hadoop.io.SequenceFile.CompressionType#NONE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SequenceFileAccessor.java    From pxf with Apache License 2.0 6 votes vote down vote up 
/**
 * Compression: based on compression codec and compression type (default
 * value RECORD). If there is no codec, compression type is ignored, and
 * NONE is used.
 *
 * @param context - container where compression codec and type are held
 */
private void getCompressionCodec(RequestContext context) {

    String userCompressCodec = context.getOption("COMPRESSION_CODEC");
    String userCompressType = context.getOption("COMPRESSION_TYPE");
    String parsedCompressType = parseCompressionType(userCompressType);

    compressionType = CompressionType.NONE;
    codec = null;
    if (userCompressCodec != null) {
        codec = codecFactory.getCodec(userCompressCodec, configuration);

        try {
            compressionType = CompressionType.valueOf(parsedCompressType);
        } catch (IllegalArgumentException e) {
            throw new IllegalArgumentException(
                    String.format("Illegal value for compression type '%s'", parsedCompressType));
        }
        LOG.debug("Compression ON: compression codec: {}, compression type: {}",
                userCompressCodec, compressionType);
    }
}
Example 2
Source File: HdfsMrsPyramidOutputFormat.java    From mrgeo with Apache License 2.0 6 votes vote down vote up 
@Override
public RecordWriter<WritableComparable<?>, Writable> getRecordWriter(TaskAttemptContext context) throws IOException
{
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(context))
  {
    // find the kind of compression to do
    compressionType = SequenceFileOutputFormat.getOutputCompressionType(context);

    // find the right codec
    codec = getCompressionCodec(context);
  }

  Path file = getDefaultWorkFile(context, "");

  MapFile.Writer out = createMapFileWriter(context, codec, compressionType, file);

  return new Writer(out);
}
Example 3
Source File: MapFileOutputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<WritableComparable, Writable> getRecordWriter(FileSystem ignored, JobConf job,
                                    String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  
  // ignore the progress parameter, since MapFile is local
  final MapFile.Writer out =
    new MapFile.Writer(job, fs, file.toString(),
                       job.getOutputKeyClass().asSubclass(WritableComparable.class),
                       job.getOutputValueClass().asSubclass(Writable.class),
                       compressionType, codec,
                       progress);

  return new RecordWriter<WritableComparable, Writable>() {

      public void write(WritableComparable key, Writable value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 4
Source File: SequenceFileOutputFormat.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<K, V> getRecordWriter(
                                        FileSystem ignored, JobConf job,
                                        String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                              job.getOutputKeyClass(),
                              job.getOutputValueClass(),
                              compressionType,
                              codec,
                              progress);

  return new RecordWriter<K, V>() {

      public void write(K key, V value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 5
Source File: MapFileOutputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<WritableComparable, Writable> getRecordWriter(FileSystem ignored, JobConf job,
                                    String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  
  // ignore the progress parameter, since MapFile is local
  final MapFile.Writer out =
    new MapFile.Writer(job, fs, file.toString(),
                       job.getOutputKeyClass().asSubclass(WritableComparable.class),
                       job.getOutputValueClass().asSubclass(Writable.class),
                       compressionType, codec,
                       progress);

  return new RecordWriter<WritableComparable, Writable>() {

      public void write(WritableComparable key, Writable value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 6
Source File: SequenceFileOutputFormat.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<K, V> getRecordWriter(
                                        FileSystem ignored, JobConf job,
                                        String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                              job.getOutputKeyClass(),
                              job.getOutputValueClass(),
                              compressionType,
                              codec,
                              progress);

  return new RecordWriter<K, V>() {

      public void write(K key, V value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 7
Source File: MapFileOutputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<WritableComparable, Writable> getRecordWriter(FileSystem ignored, JobConf job,
                                    String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  
  // ignore the progress parameter, since MapFile is local
  final MapFile.Writer out =
    new MapFile.Writer(job, fs, file.toString(),
                       job.getOutputKeyClass().asSubclass(WritableComparable.class),
                       job.getOutputValueClass().asSubclass(Writable.class),
                       compressionType, codec,
                       progress);

  return new RecordWriter<WritableComparable, Writable>() {

      public void write(WritableComparable key, Writable value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 8
Source File: SequenceFileOutputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<K, V> getRecordWriter(
                                        FileSystem ignored, JobConf job,
                                        String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                              job.getOutputKeyClass(),
                              job.getOutputValueClass(),
                              compressionType,
                              codec,
                              progress);

  return new RecordWriter<K, V>() {

      public void write(K key, V value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 9
Source File: MapFileOutputFormat.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<WritableComparable, Writable> getRecordWriter(FileSystem ignored, JobConf job,
                                    String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = SequenceFileOutputFormat.getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  
  // ignore the progress parameter, since MapFile is local
  final MapFile.Writer out =
    new MapFile.Writer(job, fs, file.toString(),
                       job.getOutputKeyClass().asSubclass(WritableComparable.class),
                       job.getOutputValueClass().asSubclass(Writable.class),
                       compressionType, codec,
                       progress);

  return new RecordWriter<WritableComparable, Writable>() {

      public void write(WritableComparable key, Writable value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 10
Source File: SequenceFileOutputFormat.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up 
public RecordWriter<K, V> getRecordWriter(
                                        FileSystem ignored, JobConf job,
                                        String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                              job.getOutputKeyClass(),
                              job.getOutputValueClass(),
                              compressionType,
                              codec,
                              progress);

  return new RecordWriter<K, V>() {

      public void write(K key, V value)
        throws IOException {

        out.append(key, value);
      }

      public void close(Reporter reporter) throws IOException { out.close();}
    };
}
Example 11
Source File: SequenceFileAsBinaryOutputFormat.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Override 
public RecordWriter <BytesWritable, BytesWritable> 
           getRecordWriter(FileSystem ignored, JobConf job,
                           String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                  getSequenceFileOutputKeyClass(job),
                  getSequenceFileOutputValueClass(job),
                  compressionType,
                  codec,
                  progress);

  return new RecordWriter<BytesWritable, BytesWritable>() {
      
      private WritableValueBytes wvaluebytes = new WritableValueBytes();

      public void write(BytesWritable bkey, BytesWritable bvalue)
        throws IOException {

        wvaluebytes.reset(bvalue);
        out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes);
        wvaluebytes.reset(null);
      }

      public void close(Reporter reporter) throws IOException { 
        out.close();
      }

    };

}
Example 12
Source File: SequenceFileAsBinaryOutputFormat.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Override 
public RecordWriter <BytesWritable, BytesWritable> 
           getRecordWriter(FileSystem ignored, JobConf job,
                           String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                  getSequenceFileOutputKeyClass(job),
                  getSequenceFileOutputValueClass(job),
                  compressionType,
                  codec,
                  progress);

  return new RecordWriter<BytesWritable, BytesWritable>() {
      
      private WritableValueBytes wvaluebytes = new WritableValueBytes();

      public void write(BytesWritable bkey, BytesWritable bvalue)
        throws IOException {

        wvaluebytes.reset(bvalue);
        out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes);
        wvaluebytes.reset(null);
      }

      public void close(Reporter reporter) throws IOException { 
        out.close();
      }

    };

}
Example 13
Source File: SequenceFileAsBinaryOutputFormat.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
@Override 
public RecordWriter <BytesWritable, BytesWritable> 
           getRecordWriter(FileSystem ignored, JobConf job,
                           String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                  getSequenceFileOutputKeyClass(job),
                  getSequenceFileOutputValueClass(job),
                  compressionType,
                  codec,
                  progress);

  return new RecordWriter<BytesWritable, BytesWritable>() {
      
      private WritableValueBytes wvaluebytes = new WritableValueBytes();

      public void write(BytesWritable bkey, BytesWritable bvalue)
        throws IOException {

        wvaluebytes.reset(bvalue);
        out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes);
        wvaluebytes.reset(null);
      }

      public void close(Reporter reporter) throws IOException { 
        out.close();
      }

    };

}
Example 14
Source File: SequenceFileAsBinaryOutputFormat.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up 
@Override 
public RecordWriter <BytesWritable, BytesWritable> 
           getRecordWriter(FileSystem ignored, JobConf job,
                           String name, Progressable progress)
  throws IOException {
  // get the path of the temporary output file 
  Path file = FileOutputFormat.getTaskOutputPath(job, name);
  
  FileSystem fs = file.getFileSystem(job);
  CompressionCodec codec = null;
  CompressionType compressionType = CompressionType.NONE;
  if (getCompressOutput(job)) {
    // find the kind of compression to do
    compressionType = getOutputCompressionType(job);

    // find the right codec
    Class<? extends CompressionCodec> codecClass = getOutputCompressorClass(job,
 DefaultCodec.class);
    codec = ReflectionUtils.newInstance(codecClass, job);
  }
  final SequenceFile.Writer out = 
    SequenceFile.createWriter(fs, job, file,
                  getSequenceFileOutputKeyClass(job),
                  getSequenceFileOutputValueClass(job),
                  compressionType,
                  codec,
                  progress);

  return new RecordWriter<BytesWritable, BytesWritable>() {
      
      private WritableValueBytes wvaluebytes = new WritableValueBytes();

      public void write(BytesWritable bkey, BytesWritable bvalue)
        throws IOException {

        wvaluebytes.reset(bvalue);
        out.appendRaw(bkey.getBytes(), 0, bkey.getLength(), wvaluebytes);
        wvaluebytes.reset(null);
      }

      public void close(Reporter reporter) throws IOException { 
        out.close();
      }

    };

}