Java Code Examples for org.apache.hadoop.io.compress.CompressionOutputStream

The following examples show how to use org.apache.hadoop.io.compress.CompressionOutputStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 2
Source Project: hadoop   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 3
Source Project: big-c   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 4
Source Project: big-c   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 5
Source Project: hbase   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
public OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor, int downStreamBufferSize)
    throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  }
  else {
    bos1 = downStream;
  }
  CompressionOutputStream cos =
      createPlainCompressionStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 6
Source Project: parquet-mr   Source File: CodecFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public BytesInput compress(BytesInput bytes) throws IOException {
  final BytesInput compressedBytes;
  if (codec == null) {
    compressedBytes = bytes;
  } else {
    compressedOutBuffer.reset();
    if (compressor != null) {
      // null compressor for non-native gzip
      compressor.reset();
    }
    CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, compressor);
    bytes.writeAllTo(cos);
    cos.finish();
    cos.close();
    compressedBytes = BytesInput.from(compressedOutBuffer);
  }
  return compressedBytes;
}
 
Example 7
Source Project: RDFS   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 8
Source Project: RDFS   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZMA codec cannot be loaded. " +
        "You may want to check LD_LIBRARY_PATH.");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzma.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 9
Source Project: RDFS   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 10
Source Project: hadoop-gpu   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 11
Source Project: hadoop-gpu   Source File: Compression.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example 12
Source Project: presto   Source File: AircompressorCompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompressedSliceOutput get()
{
    try {
        compressor.reset();
        compressedOutput.reset();
        CompressionOutputStream compressionStream = codec.createOutputStream(compressedOutput, compressor);
        return new CompressedSliceOutput(compressionStream, compressedOutput, this, () -> {});
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 
Example 13
Source Project: presto   Source File: HadoopCompressor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompressedSliceOutput get()
{
    try {
        compressor.reset();
        bufferedOutput.reset();
        CompressionOutputStream compressionStream = codec.createOutputStream(bufferedOutput, compressor);
        return new CompressedSliceOutput(compressionStream, bufferedOutput, this, () -> CodecPool.returnCompressor(compressor));
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 
Example 14
Source Project: hadoop   Source File: TestStandbyCheckpoints.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  CompressionOutputStream ret = super.createOutputStream(out);
  CompressionOutputStream spy = Mockito.spy(ret);
  Mockito.doAnswer(new GenericTestUtils.SleepAnswer(5))
    .when(spy).write(Mockito.<byte[]>any(), Mockito.anyInt(), Mockito.anyInt());
  return spy;
}
 
Example 15
Source Project: hadoop   Source File: Compression.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example 16
Source Project: big-c   Source File: TestStandbyCheckpoints.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  CompressionOutputStream ret = super.createOutputStream(out);
  CompressionOutputStream spy = Mockito.spy(ret);
  Mockito.doAnswer(new GenericTestUtils.SleepAnswer(5))
    .when(spy).write(Mockito.<byte[]>any(), Mockito.anyInt(), Mockito.anyInt());
  return spy;
}
 
Example 17
Source Project: big-c   Source File: Compression.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example 18
Source Project: hbase   Source File: Compression.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example 19
Source Project: hbase   Source File: Compression.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a compression stream without any additional wrapping into
 * buffering streams.
 */
public CompressionOutputStream createPlainCompressionStream(
    OutputStream downStream, Compressor compressor) throws IOException {
  CompressionCodec codec = getCodec(conf);
  ((Configurable)codec).getConf().setInt("io.file.buffer.size", 32 * 1024);
  return codec.createOutputStream(downStream, compressor);
}
 
Example 20
Source Project: hbase   Source File: ReusableStreamGzipCodec.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  if (ZlibFactory.isNativeZlibLoaded(getConf())) {
    return super.createOutputStream(out);
  }
  return new ReusableGzipOutputStream(out);
}
 
Example 21
Source Project: parquet-mr   Source File: TestZstandardCodec.java    License: Apache License 2.0 5 votes vote down vote up
private BytesInput compress(ZstandardCodec codec, BytesInput bytes) throws IOException {
  ByteArrayOutputStream compressedOutBuffer = new ByteArrayOutputStream((int)bytes.size());
  CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, null);
  bytes.writeAllTo(cos);
  cos.close();
  return BytesInput.from(compressedOutBuffer);
}
 
Example 22
Source Project: secor   Source File: FileReaderWriterFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
private void mockDelimitedTextFileWriter(boolean isCompressed) throws Exception {
    PowerMockito.mockStatic(FileSystem.class);
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(
            FileSystem.get(Mockito.any(URI.class),
                    Mockito.any(Configuration.class))).thenReturn(fs);

    Path fsPath = (!isCompressed) ? new Path(PATH) : new Path(PATH_GZ);

    GzipCodec codec = PowerMockito.mock(GzipCodec.class);
    PowerMockito.whenNew(GzipCodec.class).withNoArguments()
            .thenReturn(codec);

    FSDataInputStream fileInputStream = Mockito
            .mock(FSDataInputStream.class);
    FSDataOutputStream fileOutputStream = Mockito
            .mock(FSDataOutputStream.class);

    Mockito.when(fs.open(fsPath)).thenReturn(fileInputStream);
    Mockito.when(fs.create(fsPath)).thenReturn(fileOutputStream);

    CompressionInputStream inputStream = Mockito
            .mock(CompressionInputStream.class);
    CompressionOutputStream outputStream = Mockito
            .mock(CompressionOutputStream.class);
    Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
            .thenReturn(inputStream);

    Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class)))
            .thenReturn(outputStream);
}
 
Example 23
Source Project: spork   Source File: TestAllLoader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void writeTestData(File file, int recordCounts, int columnCount,
        String colSeparator) throws IOException {

    // write random test data
    GzipCodec gzipCodec = new GzipCodec();
    CompressionOutputStream out = gzipCodec
            .createOutputStream(new FileOutputStream(file));
    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
            out));

    try {

        for (int r = 0; r < recordCounts; r++) {
            // foreach row write n columns

            for (int c = 0; c < columnCount; c++) {

                if (c != 0) {
                    writer.append(colSeparator);
                }

                writer.append(String.valueOf(Math.random()));

            }
            writer.append("\n");

        }

    } finally {
        writer.close();
        out.close();
    }

}
 
Example 24
Source Project: kangaroo   Source File: SnappyFramedCodec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public CompressionOutputStream createOutputStream(final OutputStream out, final Compressor compressor)
        throws IOException {
    LOG.info("Creating compressor stream");
    return new OStreamDelegatingCompressorStream(new SnappyFramedOutputStream(out, conf.getInt(
            COMPRESSION_BLOCK_SIZE_CONF, SnappyFramedOutputStream.DEFAULT_BLOCK_SIZE),
            SnappyFramedOutputStream.DEFAULT_MIN_COMPRESSION_RATIO));
}
 
Example 25
Source Project: RDFS   Source File: SimpleSeekableFormatCodec.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  SimpleSeekableFormatOutputStream s = new SimpleSeekableFormatOutputStream(out);
  s.setConf(getConf());
  return s;
}
 
Example 26
Source Project: RDFS   Source File: Compression.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example 27
Source Project: hadoop-gpu   Source File: Compression.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example 28
Source Project: pxf   Source File: NotSoNiceCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
        throws IOException {
    return null;
}
 
Example 29
Source Project: pxf   Source File: NotSoNiceCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out,
                                                  Compressor compressor) throws IOException {
    return null;
}
 
Example 30
Source Project: sylph   Source File: NoneCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out, Compressor compressor)
        throws IOException
{
    return this.createOutputStream(out);
}