Java Code Examples for org.apache.hadoop.io.compress.CompressionCodec#createCompressor()

The following examples show how to use org.apache.hadoop.io.compress.CompressionCodec#createCompressor() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CodecPool.java    From tajo with Apache License 2.0 5 votes vote down vote up 
/**
 * Get a {@link Compressor} for the given {@link CompressionCodec} from the
 * pool or a new one.
 *
 * @param codec
 *          the <code>CompressionCodec</code> for which to get the
 *          <code>Compressor</code>
 * @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
 * @return <code>Compressor</code> for the given <code>CompressionCodec</code>
 *         from the pool or a new one
 */
public static Compressor getCompressor(CompressionCodec codec, Configuration conf) {
  Compressor compressor = borrow(COMPRESSOR_POOL, codec.getCompressorType());
  if (compressor == null) {
    compressor = codec.createCompressor();
    LOG.info("Got brand-new compressor ["+codec.getDefaultExtension()+"]");
  } else {
    compressor.reinit(conf);
    if(LOG.isDebugEnabled()) {
      LOG.debug("Got recycled compressor");
    }
  }
  return compressor;
}
Example 2
Source File: CodecPool.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up 
/**
 * Get a {@link Compressor} for the given {@link CompressionCodec} from the
 * pool or a new one.
 *
 * @param codec
 *          the <code>CompressionCodec</code> for which to get the
 *          <code>Compressor</code>
 * @param conf the <code>Configuration</code> object which contains confs for creating or reinit the compressor
 * @return <code>Compressor</code> for the given <code>CompressionCodec</code>
 *         from the pool or a new one
 */
public static Compressor getCompressor(CompressionCodec codec, Configuration conf) {
  Compressor compressor = borrow(COMPRESSOR_POOL, codec.getCompressorType());
  if (compressor == null) {
    compressor = codec.createCompressor();
    LOG.info("Got brand-new compressor ["+codec.getDefaultExtension()+"]");
  } else {
    compressor.reinit(conf);
    if(LOG.isDebugEnabled()) {
      LOG.debug("Got recycled compressor");
    }
  }
  return compressor;
}
Example 3
Source File: AircompressorCompressor.java    From presto with Apache License 2.0 4 votes vote down vote up 
public AircompressorCompressedSliceOutputSupplier(CompressionCodec codec, int minChunkSize, int maxChunkSize)
{
    this.codec = requireNonNull(codec, "codec is null");
    this.compressor = codec.createCompressor();
    this.compressedOutput = new ChunkedSliceOutput(minChunkSize, maxChunkSize);
}
Example 4
Source File: DirectCodecFactory.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
private CodecPool(final CompressionCodec codec){
  try {
    boolean supportDirectDecompressor = codec.getClass() == DIRECT_DECOMPRESSION_CODEC_CLASS;
    compressorPool = new GenericObjectPool(new BasePoolableObjectFactory() {
      public Object makeObject() throws Exception {
        return codec.createCompressor();
      }
    }, Integer.MAX_VALUE);

    Object com = compressorPool.borrowObject();
    if (com != null) {
      cPools.put(com.getClass(), compressorPool);
      compressorPool.returnObject(com);
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor", codec.getClass().getName()));
      }
    }

    decompressorPool = new GenericObjectPool(new BasePoolableObjectFactory() {
      public Object makeObject() throws Exception {
        return codec.createDecompressor();
      }
    }, Integer.MAX_VALUE);

    Object decom = decompressorPool.borrowObject();
    if (decom != null) {
      dePools.put(decom.getClass(), decompressorPool);
      decompressorPool.returnObject(decom);
    } else {
      if (LOG.isDebugEnabled()) {
        LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "decompressor", codec.getClass().getName()));
      }
    }

    if (supportDirectDecompressor) {
      directDecompressorPool = new GenericObjectPool(
          new BasePoolableObjectFactory() {
            public Object makeObject() throws Exception {
              return CREATE_DIRECT_DECOMPRESSOR_METHOD.invoke(DIRECT_DECOMPRESSION_CODEC_CLASS);
            }
          }, Integer.MAX_VALUE);

      Object ddecom = directDecompressorPool.borrowObject();
      if (ddecom != null) {
        directDePools.put(ddecom.getClass(), directDecompressorPool);
        directDecompressorPool.returnObject(ddecom);

      } else {
        supportDirectDecompressor = false;
        if (LOG.isDebugEnabled()) {
          LOG.debug(String.format(BYTE_BUF_IMPL_NOT_FOUND_MSG, "compressor", codec.getClass().getName()));
        }
      }

    } else {
      directDecompressorPool = null;
    }

    this.supportDirectDecompressor = supportDirectDecompressor;
  } catch (Exception e) {
    throw new ParquetCompressionCodecException("Error creating compression codec pool.", e);
  }
}
Example 5
Source File: DataSegmentWriter.java    From RDFS with Apache License 2.0 4 votes vote down vote up 
/**
 * Create a new data segment from uncompressed data and a codec.
 * This is called by the writer.
 * @param compressor  for reusing a compressor.  It can be null.
 */
DataSegmentWriter(SimpleSeekableFormat.Buffer uncompressedData,
    CompressionCodec codec,
    Compressor compressor) throws IOException {

  // Try compress
  if (codec != null) {
    SimpleSeekableFormat.Buffer compressedData = new SimpleSeekableFormat.Buffer();
    OutputStream out;
    if (compressor == null) {
      compressor = codec.createCompressor();
    } else {
      compressor.reset();
    }
    out = codec.createOutputStream(compressedData, compressor);
    out.write(uncompressedData.getData(), 0, uncompressedData.getLength());
    out.close();

    // Don't compress if the result is longer than uncompressed data.
    if (compressedData.getLength() + codec.getClass().getName().length()
        < uncompressedData.getLength() + 8) {
      codecName = codec.getClass().getName();
      storedData = compressedData;
    } else {
      codecName = "";
      storedData = uncompressedData;
    }
  } else {
    // no compression
    codecName = "";
    storedData = uncompressedData;
  }
  codecNameUTF8 = getCodecNameUTF8(codecName);

  // Calculate CRC32 only when there are no compression.
  if (codecName.length() == 0) {
    CRC32 crc32 = new CRC32();
    crc32.update(storedData.getData(), 0, storedData.getLength());
    crc32Value = crc32.getValue();
  } else {
    crc32Value = 0;
  }
}