Java Code Examples for org.apache.hadoop.io.compress.Compressor#reset()
The following examples show how to use
org.apache.hadoop.io.compress.Compressor#reset() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Compression.java From hadoop with Apache License 2.0 | 6 votes |
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a compressor: " + compressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
Example 2
Source File: Compression.java From big-c with Apache License 2.0 | 6 votes |
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { if(LOG.isDebugEnabled()) { LOG.debug("Got a compressor: " + compressor.hashCode()); } } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
Example 3
Source File: Compression.java From hbase with Apache License 2.0 | 6 votes |
public Compressor getCompressor() { CompressionCodec codec = getCodec(conf); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (LOG.isTraceEnabled()) LOG.trace("Retrieved compressor " + compressor + " from pool."); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using it. LOG.warn("Compressor obtained from CodecPool is already finished()"); } compressor.reset(); } return compressor; } return null; }
Example 4
Source File: Compression.java From RDFS with Apache License 2.0 | 6 votes |
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { LOG.debug("Got a compressor: " + compressor.hashCode()); } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
Example 5
Source File: Compression.java From hadoop-gpu with Apache License 2.0 | 6 votes |
public Compressor getCompressor() throws IOException { CompressionCodec codec = getCodec(); if (codec != null) { Compressor compressor = CodecPool.getCompressor(codec); if (compressor != null) { if (compressor.finished()) { // Somebody returns the compressor to CodecPool but is still using // it. LOG.warn("Compressor obtained from CodecPool already finished()"); } else { LOG.debug("Got a compressor: " + compressor.hashCode()); } /** * Following statement is necessary to get around bugs in 0.18 where a * compressor is referenced after returned back to the codec pool. */ compressor.reset(); } return compressor; } return null; }
Example 6
Source File: CodecPool.java From tajo with Apache License 2.0 | 5 votes |
/** * Return the {@link Compressor} to the pool. * * @param compressor * the <code>Compressor</code> to be returned to the pool */ public static void returnCompressor(Compressor compressor) { if (compressor == null) { return; } // if the compressor can't be reused, don't pool it. if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) { return; } compressor.reset(); payback(COMPRESSOR_POOL, compressor); }
Example 7
Source File: EncodedDataBlock.java From hbase with Apache License 2.0 | 5 votes |
/** * Find the size of compressed data assuming that buffer will be compressed * using given algorithm. * @param algo compression algorithm * @param compressor compressor already requested from codec * @param inputBuffer Array to be compressed. * @param offset Offset to beginning of the data. * @param length Length to be compressed. * @return Size of compressed data in bytes. * @throws IOException */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH_EXCEPTION", justification="No sure what findbugs wants but looks to me like no NPE") public static int getCompressedSize(Algorithm algo, Compressor compressor, byte[] inputBuffer, int offset, int length) throws IOException { // Create streams // Storing them so we can close them final IOUtils.NullOutputStream nullOutputStream = new IOUtils.NullOutputStream(); final DataOutputStream compressedStream = new DataOutputStream(nullOutputStream); OutputStream compressingStream = null; try { if (compressor != null) { compressor.reset(); } compressingStream = algo.createCompressionStream(compressedStream, compressor, 0); compressingStream.write(inputBuffer, offset, length); compressingStream.flush(); return compressedStream.size(); } finally { nullOutputStream.close(); compressedStream.close(); if (compressingStream != null) { compressingStream.close(); } } }
Example 8
Source File: CodecPool.java From incubator-tajo with Apache License 2.0 | 5 votes |
/** * Return the {@link Compressor} to the pool. * * @param compressor * the <code>Compressor</code> to be returned to the pool */ public static void returnCompressor(Compressor compressor) { if (compressor == null) { return; } // if the compressor can't be reused, don't pool it. if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) { return; } compressor.reset(); payback(COMPRESSOR_POOL, compressor); }
Example 9
Source File: DataSegmentWriter.java From RDFS with Apache License 2.0 | 4 votes |
/** * Create a new data segment from uncompressed data and a codec. * This is called by the writer. * @param compressor for reusing a compressor. It can be null. */ DataSegmentWriter(SimpleSeekableFormat.Buffer uncompressedData, CompressionCodec codec, Compressor compressor) throws IOException { // Try compress if (codec != null) { SimpleSeekableFormat.Buffer compressedData = new SimpleSeekableFormat.Buffer(); OutputStream out; if (compressor == null) { compressor = codec.createCompressor(); } else { compressor.reset(); } out = codec.createOutputStream(compressedData, compressor); out.write(uncompressedData.getData(), 0, uncompressedData.getLength()); out.close(); // Don't compress if the result is longer than uncompressed data. if (compressedData.getLength() + codec.getClass().getName().length() < uncompressedData.getLength() + 8) { codecName = codec.getClass().getName(); storedData = compressedData; } else { codecName = ""; storedData = uncompressedData; } } else { // no compression codecName = ""; storedData = uncompressedData; } codecNameUTF8 = getCodecNameUTF8(codecName); // Calculate CRC32 only when there are no compression. if (codecName.length() == 0) { CRC32 crc32 = new CRC32(); crc32.update(storedData.getData(), 0, storedData.getLength()); crc32Value = crc32.getValue(); } else { crc32Value = 0; } }