Java Code Examples for org.apache.hadoop.io.compress.Compressor#reset()

public Compressor getCompressor() throws IOException {
  CompressionCodec codec = getCodec();
  if (codec != null) {
    Compressor compressor = CodecPool.getCompressor(codec);
    if (compressor != null) {
      if (compressor.finished()) {
        // Somebody returns the compressor to CodecPool but is still using
        // it.
        LOG.warn("Compressor obtained from CodecPool already finished()");
      } else {
        if(LOG.isDebugEnabled()) {
          LOG.debug("Got a compressor: " + compressor.hashCode());
        }
      }
      /**
       * Following statement is necessary to get around bugs in 0.18 where a
       * compressor is referenced after returned back to the codec pool.
       */
      compressor.reset();
    }
    return compressor;
  }
  return null;
}
 

public Compressor getCompressor() throws IOException {
  CompressionCodec codec = getCodec();
  if (codec != null) {
    Compressor compressor = CodecPool.getCompressor(codec);
    if (compressor != null) {
      if (compressor.finished()) {
        // Somebody returns the compressor to CodecPool but is still using
        // it.
        LOG.warn("Compressor obtained from CodecPool already finished()");
      } else {
        if(LOG.isDebugEnabled()) {
          LOG.debug("Got a compressor: " + compressor.hashCode());
        }
      }
      /**
       * Following statement is necessary to get around bugs in 0.18 where a
       * compressor is referenced after returned back to the codec pool.
       */
      compressor.reset();
    }
    return compressor;
  }
  return null;
}
 

public Compressor getCompressor() {
  CompressionCodec codec = getCodec(conf);
  if (codec != null) {
    Compressor compressor = CodecPool.getCompressor(codec);
    if (LOG.isTraceEnabled()) LOG.trace("Retrieved compressor " + compressor + " from pool.");
    if (compressor != null) {
      if (compressor.finished()) {
        // Somebody returns the compressor to CodecPool but is still using it.
        LOG.warn("Compressor obtained from CodecPool is already finished()");
      }
      compressor.reset();
    }
    return compressor;
  }
  return null;
}
 

public Compressor getCompressor() throws IOException {
  CompressionCodec codec = getCodec();
  if (codec != null) {
    Compressor compressor = CodecPool.getCompressor(codec);
    if (compressor != null) {
      if (compressor.finished()) {
        // Somebody returns the compressor to CodecPool but is still using
        // it.
        LOG.warn("Compressor obtained from CodecPool already finished()");
      } else {
        LOG.debug("Got a compressor: " + compressor.hashCode());
      }
      /**
       * Following statement is necessary to get around bugs in 0.18 where a
       * compressor is referenced after returned back to the codec pool.
       */
      compressor.reset();
    }
    return compressor;
  }
  return null;
}
 

public Compressor getCompressor() throws IOException {
  CompressionCodec codec = getCodec();
  if (codec != null) {
    Compressor compressor = CodecPool.getCompressor(codec);
    if (compressor != null) {
      if (compressor.finished()) {
        // Somebody returns the compressor to CodecPool but is still using
        // it.
        LOG.warn("Compressor obtained from CodecPool already finished()");
      } else {
        LOG.debug("Got a compressor: " + compressor.hashCode());
      }
      /**
       * Following statement is necessary to get around bugs in 0.18 where a
       * compressor is referenced after returned back to the codec pool.
       */
      compressor.reset();
    }
    return compressor;
  }
  return null;
}
 

/**
 * Return the {@link Compressor} to the pool.
 *
 * @param compressor
 *          the <code>Compressor</code> to be returned to the pool
 */
public static void returnCompressor(Compressor compressor) {
  if (compressor == null) {
    return;
  }
  // if the compressor can't be reused, don't pool it.
  if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) {
    return;
  }
  compressor.reset();
  payback(COMPRESSOR_POOL, compressor);
}
 

/**
 * Find the size of compressed data assuming that buffer will be compressed
 * using given algorithm.
 * @param algo compression algorithm
 * @param compressor compressor already requested from codec
 * @param inputBuffer Array to be compressed.
 * @param offset Offset to beginning of the data.
 * @param length Length to be compressed.
 * @return Size of compressed data in bytes.
 * @throws IOException
 */
@edu.umd.cs.findbugs.annotations.SuppressWarnings(value="NP_NULL_ON_SOME_PATH_EXCEPTION",
     justification="No sure what findbugs wants but looks to me like no NPE")
public static int getCompressedSize(Algorithm algo, Compressor compressor,
    byte[] inputBuffer, int offset, int length) throws IOException {

  // Create streams
  // Storing them so we can close them
  final IOUtils.NullOutputStream nullOutputStream = new IOUtils.NullOutputStream();
  final DataOutputStream compressedStream = new DataOutputStream(nullOutputStream);
  OutputStream compressingStream = null;


  try {
    if (compressor != null) {
      compressor.reset();
    }

    compressingStream = algo.createCompressionStream(compressedStream, compressor, 0);

    compressingStream.write(inputBuffer, offset, length);
    compressingStream.flush();

    return compressedStream.size();
  } finally {
    nullOutputStream.close();
    compressedStream.close();
    if (compressingStream != null) {
      compressingStream.close();
    }
  }
}
 

/**
 * Return the {@link Compressor} to the pool.
 *
 * @param compressor
 *          the <code>Compressor</code> to be returned to the pool
 */
public static void returnCompressor(Compressor compressor) {
  if (compressor == null) {
    return;
  }
  // if the compressor can't be reused, don't pool it.
  if (compressor.getClass().isAnnotationPresent(DoNotPool.class)) {
    return;
  }
  compressor.reset();
  payback(COMPRESSOR_POOL, compressor);
}
 

/**
 * Create a new data segment from uncompressed data and a codec.
 * This is called by the writer.
 * @param compressor  for reusing a compressor.  It can be null.
 */
DataSegmentWriter(SimpleSeekableFormat.Buffer uncompressedData,
    CompressionCodec codec,
    Compressor compressor) throws IOException {

  // Try compress
  if (codec != null) {
    SimpleSeekableFormat.Buffer compressedData = new SimpleSeekableFormat.Buffer();
    OutputStream out;
    if (compressor == null) {
      compressor = codec.createCompressor();
    } else {
      compressor.reset();
    }
    out = codec.createOutputStream(compressedData, compressor);
    out.write(uncompressedData.getData(), 0, uncompressedData.getLength());
    out.close();

    // Don't compress if the result is longer than uncompressed data.
    if (compressedData.getLength() + codec.getClass().getName().length()
        < uncompressedData.getLength() + 8) {
      codecName = codec.getClass().getName();
      storedData = compressedData;
    } else {
      codecName = "";
      storedData = uncompressedData;
    }
  } else {
    // no compression
    codecName = "";
    storedData = uncompressedData;
  }
  codecNameUTF8 = getCodecNameUTF8(codecName);

  // Calculate CRC32 only when there are no compression.
  if (codecName.length() == 0) {
    CRC32 crc32 = new CRC32();
    crc32.update(storedData.getData(), 0, storedData.getLength());
    crc32Value = crc32.getValue();
  } else {
    crc32Value = 0;
  }
}