org.apache.hadoop.io.compress.CompressionOutputStream Java Examples

The following examples show how to use org.apache.hadoop.io.compress.CompressionOutputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Compression.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #2
Source File: Compression.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #3
Source File: Compression.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #4
Source File: Compression.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZMA codec cannot be loaded. " +
        "You may want to check LD_LIBRARY_PATH.");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzma.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #5
Source File: Compression.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #6
Source File: Compression.java    From hbase with Apache License 2.0 6 votes vote down vote up
public OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor, int downStreamBufferSize)
    throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  }
  else {
    bos1 = downStream;
  }
  CompressionOutputStream cos =
      createPlainCompressionStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #7
Source File: Compression.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #8
Source File: Compression.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #9
Source File: Compression.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  codec.getConf().setInt("io.file.buffer.size", 32 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #10
Source File: Compression.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized OutputStream createCompressionStream(
    OutputStream downStream, Compressor compressor,
    int downStreamBufferSize) throws IOException {
  if (!isSupported()) {
    throw new IOException(
        "LZO codec class not specified. Did you forget to set property "
            + CONF_LZO_CLASS + "?");
  }
  OutputStream bos1 = null;
  if (downStreamBufferSize > 0) {
    bos1 = new BufferedOutputStream(downStream, downStreamBufferSize);
  } else {
    bos1 = downStream;
  }
  conf.setInt("io.compression.codec.lzo.buffersize", 64 * 1024);
  CompressionOutputStream cos =
      codec.createOutputStream(bos1, compressor);
  BufferedOutputStream bos2 =
      new BufferedOutputStream(new FinishOnFlushCompressionStream(cos),
          DATA_OBUF_SIZE);
  return bos2;
}
 
Example #11
Source File: CodecFactory.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
@Override
public BytesInput compress(BytesInput bytes) throws IOException {
  final BytesInput compressedBytes;
  if (codec == null) {
    compressedBytes = bytes;
  } else {
    compressedOutBuffer.reset();
    if (compressor != null) {
      // null compressor for non-native gzip
      compressor.reset();
    }
    CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, compressor);
    bytes.writeAllTo(cos);
    cos.finish();
    cos.close();
    compressedBytes = BytesInput.from(compressedOutBuffer);
  }
  return compressedBytes;
}
 
Example #12
Source File: Compression.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example #13
Source File: Compression.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a compression stream without any additional wrapping into
 * buffering streams.
 */
public CompressionOutputStream createPlainCompressionStream(
    OutputStream downStream, Compressor compressor) throws IOException {
  CompressionCodec codec = getCodec(conf);
  ((Configurable)codec).getConf().setInt("io.file.buffer.size", 32 * 1024);
  return codec.createOutputStream(downStream, compressor);
}
 
Example #14
Source File: AircompressorCompressor.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public CompressedSliceOutput get()
{
    try {
        compressor.reset();
        compressedOutput.reset();
        CompressionOutputStream compressionStream = codec.createOutputStream(compressedOutput, compressor);
        return new CompressedSliceOutput(compressionStream, compressedOutput, this, () -> {});
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 
Example #15
Source File: ReusableStreamGzipCodec.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  if (ZlibFactory.isNativeZlibLoaded(getConf())) {
    return super.createOutputStream(out);
  }
  return new ReusableGzipOutputStream(out);
}
 
Example #16
Source File: TestZstandardCodec.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private BytesInput compress(ZstandardCodec codec, BytesInput bytes) throws IOException {
  ByteArrayOutputStream compressedOutBuffer = new ByteArrayOutputStream((int)bytes.size());
  CompressionOutputStream cos = codec.createOutputStream(compressedOutBuffer, null);
  bytes.writeAllTo(cos);
  cos.close();
  return BytesInput.from(compressedOutBuffer);
}
 
Example #17
Source File: FileReaderWriterFactoryTest.java    From secor with Apache License 2.0 5 votes vote down vote up
private void mockDelimitedTextFileWriter(boolean isCompressed) throws Exception {
    PowerMockito.mockStatic(FileSystem.class);
    FileSystem fs = Mockito.mock(FileSystem.class);
    Mockito.when(
            FileSystem.get(Mockito.any(URI.class),
                    Mockito.any(Configuration.class))).thenReturn(fs);

    Path fsPath = (!isCompressed) ? new Path(PATH) : new Path(PATH_GZ);

    GzipCodec codec = PowerMockito.mock(GzipCodec.class);
    PowerMockito.whenNew(GzipCodec.class).withNoArguments()
            .thenReturn(codec);

    FSDataInputStream fileInputStream = Mockito
            .mock(FSDataInputStream.class);
    FSDataOutputStream fileOutputStream = Mockito
            .mock(FSDataOutputStream.class);

    Mockito.when(fs.open(fsPath)).thenReturn(fileInputStream);
    Mockito.when(fs.create(fsPath)).thenReturn(fileOutputStream);

    CompressionInputStream inputStream = Mockito
            .mock(CompressionInputStream.class);
    CompressionOutputStream outputStream = Mockito
            .mock(CompressionOutputStream.class);
    Mockito.when(codec.createInputStream(Mockito.any(InputStream.class)))
            .thenReturn(inputStream);

    Mockito.when(codec.createOutputStream(Mockito.any(OutputStream.class)))
            .thenReturn(outputStream);
}
 
Example #18
Source File: TestAllLoader.java    From spork with Apache License 2.0 5 votes vote down vote up
@Override
public void writeTestData(File file, int recordCounts, int columnCount,
        String colSeparator) throws IOException {

    // write random test data
    GzipCodec gzipCodec = new GzipCodec();
    CompressionOutputStream out = gzipCodec
            .createOutputStream(new FileOutputStream(file));
    BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(
            out));

    try {

        for (int r = 0; r < recordCounts; r++) {
            // foreach row write n columns

            for (int c = 0; c < columnCount; c++) {

                if (c != 0) {
                    writer.append(colSeparator);
                }

                writer.append(String.valueOf(Math.random()));

            }
            writer.append("\n");

        }

    } finally {
        writer.close();
        out.close();
    }

}
 
Example #19
Source File: SnappyFramedCodec.java    From kangaroo with Apache License 2.0 5 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public CompressionOutputStream createOutputStream(final OutputStream out, final Compressor compressor)
        throws IOException {
    LOG.info("Creating compressor stream");
    return new OStreamDelegatingCompressorStream(new SnappyFramedOutputStream(out, conf.getInt(
            COMPRESSION_BLOCK_SIZE_CONF, SnappyFramedOutputStream.DEFAULT_BLOCK_SIZE),
            SnappyFramedOutputStream.DEFAULT_MIN_COMPRESSION_RATIO));
}
 
Example #20
Source File: SimpleSeekableFormatCodec.java    From RDFS with Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  SimpleSeekableFormatOutputStream s = new SimpleSeekableFormatOutputStream(out);
  s.setConf(getConf());
  return s;
}
 
Example #21
Source File: Compression.java    From RDFS with Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example #22
Source File: Compression.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example #23
Source File: HadoopCompressor.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public CompressedSliceOutput get()
{
    try {
        compressor.reset();
        bufferedOutput.reset();
        CompressionOutputStream compressionStream = codec.createOutputStream(bufferedOutput, compressor);
        return new CompressedSliceOutput(compressionStream, bufferedOutput, this, () -> CodecPool.returnCompressor(compressor));
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 
Example #24
Source File: TestStandbyCheckpoints.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  CompressionOutputStream ret = super.createOutputStream(out);
  CompressionOutputStream spy = Mockito.spy(ret);
  Mockito.doAnswer(new GenericTestUtils.SleepAnswer(5))
    .when(spy).write(Mockito.<byte[]>any(), Mockito.anyInt(), Mockito.anyInt());
  return spy;
}
 
Example #25
Source File: Compression.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example #26
Source File: Compression.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void flush() throws IOException {
  CompressionOutputStream cout = (CompressionOutputStream) out;
  cout.finish();
  cout.flush();
  cout.resetState();
}
 
Example #27
Source File: TestStandbyCheckpoints.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  CompressionOutputStream ret = super.createOutputStream(out);
  CompressionOutputStream spy = Mockito.spy(ret);
  Mockito.doAnswer(new GenericTestUtils.SleepAnswer(5))
    .when(spy).write(Mockito.<byte[]>any(), Mockito.anyInt(), Mockito.anyInt());
  return spy;
}
 
Example #28
Source File: SnappyFramedCodec.java    From kangaroo with Apache License 2.0 4 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
public CompressionOutputStream createOutputStream(final OutputStream out) throws IOException {
    return createOutputStream(out, null);
}
 
Example #29
Source File: TestMapFile.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out)
    throws IOException {
  return mock(CompressionOutputStream.class);
}
 
Example #30
Source File: TestBloomMapFile.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public CompressionOutputStream createOutputStream(OutputStream out,
    Compressor compressor) throws IOException {
  return mock(CompressionOutputStream.class);
}