org.xerial.snappy.Snappy Java Examples

@Override
public byte[] compress(
    final byte[] data ,
    final int start ,
    final int length ,
    final CompressResult compressResult ) throws IOException {
  byte[] compressTarget;
  if ( start != 0 ) {
    compressTarget = new byte[length];
    System.arraycopy( data , start , compressTarget , 0 , length );
  } else {
    compressTarget = data;
  }

  byte[] compressByte = Snappy.rawCompress( compressTarget , length );
  byte[] retVal = new byte[ Integer.BYTES + compressByte.length ];
  ByteBuffer wrapBuffer = ByteBuffer.wrap( retVal );
  wrapBuffer.putInt( length );
  wrapBuffer.put( compressByte );

  compressResult.feedBack( length , compressByte.length );

  return retVal;
}
 

@Test
public void shouldCompressSmallContent() throws Exception {
    String text = "Lorem ipsum dolor sit amet, consectetuer adipiscing elit.";
    text += text;

    channel.pipeline().addFirst(new SnappyFeatureHandler());

    ByteBuf content = Unpooled.copiedBuffer(text, CharsetUtil.UTF_8);

    UpsertRequest request = new UpsertRequest("key", content.copy(), "bucket");
    request.partition((short) 512);
    channel.writeOutbound(request);
    FullBinaryMemcacheRequest outbound = (FullBinaryMemcacheRequest) channel.readOutbound();
    assertNotNull(outbound);

    assertEquals(KeyValueHandler.DATATYPE_SNAPPY, outbound.getDataType());

    byte[] compressed = new byte[outbound.content().readableBytes()];
    outbound.content().getBytes(0, compressed);
    byte[] uncompressed = Snappy.uncompress(compressed);
    assertArrayEquals(text.getBytes(CharsetUtil.UTF_8), uncompressed);
    ReferenceCountUtil.release(outbound);
}
 

@Test
public void shouldDecompressSmallContent() throws Exception {
    String text = "Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo " +
        "ligula eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient " +
        "montes, nascetur ridiculus mus.";

    channel.pipeline().addFirst(new SnappyFeatureHandler());

    ByteBuf content = Unpooled.wrappedBuffer(Snappy.compress(text.getBytes(CharsetUtil.UTF_8)));
    FullBinaryMemcacheResponse response = new DefaultFullBinaryMemcacheResponse(KEY, Unpooled.EMPTY_BUFFER,
        content);
    response.setDataType(KeyValueHandler.DATATYPE_SNAPPY);

    GetRequest requestMock = mock(GetRequest.class);
    requestQueue.add(requestMock);
    channel.writeInbound(response);

    GetResponse event = (GetResponse) eventSink.responseEvents().get(0).getMessage();
    assertEquals(text, event.content().toString(CHARSET));
    ReferenceCountUtil.release(event);
}
 

@Override
public ByteBuf encode(ByteBuf source) {
    int uncompressedLength = source.readableBytes();
    int maxLength = Snappy.maxCompressedLength(uncompressedLength);

    ByteBuffer sourceNio = source.nioBuffer(source.readerIndex(), source.readableBytes());

    ByteBuf target = PooledByteBufAllocator.DEFAULT.buffer(maxLength, maxLength);
    ByteBuffer targetNio = target.nioBuffer(0, maxLength);

    int compressedLength = 0;
    try {
        compressedLength = Snappy.compress(sourceNio, targetNio);
    } catch (IOException e) {
        log.error("Failed to compress to Snappy: {}", e.getMessage());
    }
    target.writerIndex(compressedLength);
    return target;
}
 

public Frame compress(Frame frame) throws IOException
{
    byte[] input = CBUtil.readRawBytes(frame.body);
    ByteBuf output = CBUtil.allocator.heapBuffer(Snappy.maxCompressedLength(input.length));

    try
    {
        int written = Snappy.compress(input, 0, input.length, output.array(), output.arrayOffset());
        output.writerIndex(written);
    }
    catch (final Throwable e)
    {
        output.release();
        throw e;
    }
    finally
    {
        //release the old frame
        frame.release();
    }

    return frame.with(output);
}
 

public byte[] objectToBytes(Object o) throws IOException {
    try {
        byte[] bytes = MarshallerUtil.obj2bytes(marshaller, o);

        if( DEBUG )
            log.debug( StackTraceUtil.getStackTrace(Thread.currentThread().getStackTrace()) );

        if( USE_SNAPPY_COMPRESSION ) {
            byte[] compressBuf = Snappy.compress(bytes);
            if( DEBUG ) {
                log.debug("KhanGridMarshaller/SIZE=" + bytes.length + "/COMPRESS=" + compressBuf.length);
            }
            return compressBuf;
        } else {
            if( DEBUG ) {
                log.debug("KhanGridMarshaller/SIZE=" + bytes.length );
            }
            return bytes;
        }
    } catch (Exception e) {
        throw new IOException("Exception");
    }
}
 

@Override
public byte[] decompress(byte[] input) {
  try {
    return Snappy.uncompress(input);
  } catch (IOException e) {
    throw new CompressionException(e);
  }
}
 

private static void serializeDoubleArray(final DataOutput out, final double[] val, boolean compress)
    throws IOException {
  if (compress && val.length > 250) {
    out.write(DOUBLE_ARRAY_C);
    byte[] b = Snappy.compress(val);
    LongPacker.packInt(out, b.length);
    out.write(b);
  } else {
    out.write(DOUBLE_ARRAY);
    LongPacker.packInt(out, val.length);
    for (double s : val) {
      out.writeDouble(s);
    }
  }
}
 

@Override
public byte[] compress(byte[] input) {
  try {
    return Snappy.compress(input);
  } catch (IOException e) {
    throw new CompressionException(e);
  }
}
 

/**
 * Create a new instance of the SnappyCompressor.
 * 
 * @throws IllegaltStateException when the Snappy native library is unavailable
 */
@SuppressFBWarnings(value="DMI_RANDOM_USED_ONLY_ONCE", justification="The native library is only used once so we only need to generate one random number")
public SnappyCompressor() {
  synchronized (defaultInstance) {
    if (!nativeLibraryLoaded) {
      try {
        SnappyUtils.setSnappySystemProperties(SnappyUtils.getOrCreateLogWriter());
        Snappy.getNativeLibraryVersion();
      } catch (SnappyError se) {
        throw new IllegalStateException(LocalizedStrings.SnappyCompressor_UNABLE_TO_LOAD_NATIVE_SNAPPY_LIBRARY.toLocalizedString(), se);
      }
      nativeLibraryLoaded = true;
    }
  }
}
 

@Override
public AsyncFuture<Void> consume(final byte[] message) throws ConsumerSchemaException {
  final List<Spotify100.Metric> metrics;
  try {
    metrics = Spotify100.Batch.parseFrom(Snappy.uncompress(message)).getMetricList();
  } catch (IOException e) {
    throw new ConsumerSchemaValidationException("Invalid batch of metrics", e);
  }

  final List<AsyncFuture<Ingestion>> ingestions = new ArrayList<>();
  for (Spotify100.Metric metric : metrics) {

    if (metric.getTime() <= 0) {
      throw new ConsumerSchemaValidationException(
        "time: field must be a positive number: " + metric.toString());
    }

    final Series s = Series.of(metric.getKey(), metric.getTagsMap(), metric.getResourceMap());
    final Point p = new Point(metric.getTime(), metric.getValue());
    final List<Point> points = ImmutableList.of(p);

    reporter.reportMessageDrift(clock.currentTimeMillis() - p.getTimestamp());

    ingestions.add(ingestion.write(new Request(s, MetricCollection.points(points))));
  }

  reporter.reportMetricsIn(metrics.size());

  // Return Void future, to not leak unnecessary information from the backend but just
  // allow monitoring of when the consumption is done.
  return async.collectAndDiscard(ingestions);
}
 

@Override
public ByteBuf decode(ByteBuf encoded, int uncompressedLength) throws IOException {
    ByteBuf uncompressed = PooledByteBufAllocator.DEFAULT.buffer(uncompressedLength, uncompressedLength);
    ByteBuffer uncompressedNio = uncompressed.nioBuffer(0, uncompressedLength);

    ByteBuffer encodedNio = encoded.nioBuffer(encoded.readerIndex(), encoded.readableBytes());
    Snappy.uncompress(encodedNio, uncompressedNio);

    uncompressed.writerIndex(uncompressedLength);
    return uncompressed;
}
 

@Test
public void testTimeValidationError() throws Exception {
  exceptionRule.expect(ConsumerSchemaValidationException.class);
  exceptionRule.expectMessage("time: field must be a positive number");

  final Metric metric = Metric.newBuilder().setTime(-1542830480000L).build();
  final Batch batch = Batch.newBuilder().addMetric(metric).build();

  consumer.consume(Snappy.compress(batch.toByteArray()));
}
 

private static void serializeCharArray(final DataOutput out, final char[] val, boolean compress)
    throws IOException {
  if (compress && val.length > 250) {
    out.write(CHAR_ARRAY_C);
    byte[] b = Snappy.compress(val);
    LongPacker.packInt(out, b.length);
    out.write(b);
  } else {
    out.write(CHAR_ARRAY);
    LongPacker.packInt(out, val.length);
    for (char s : val) {
      out.writeChar(s);
    }
  }
}
 

public static byte[] compressFilter(byte[] datas) {
    if (Constants.COMPRESS_FILE) {
        try {
            datas = Snappy.compress(datas);
        } catch (IOException e) {
            log.error("Compress error!", e);
        }
    }
    return datas;
}
 

public static byte[] uncompressFilter(byte[] datas) {
    if (Constants.COMPRESS_FILE) {
        try {
            datas = Snappy.uncompress(datas);
        } catch (IOException e) {
            log.error("Uncompress error!", e);
        }
    }
    return datas;
}
 

@Test
public void testEmptyKeyIsConsumed() throws Exception {
  final Metric metric = Metric.newBuilder().setTime(1000L).build();
  final Batch batch = Batch.newBuilder().addMetric(metric).build();

  consumer.consume(Snappy.compress((batch.toByteArray())));

  final Series s = Series.of(metric.getKey(), metric.getTagsMap(), metric.getResourceMap());
  final Point p = new Point(metric.getTime(), metric.getValue());
  final List<Point> points = ImmutableList.of(p);
  verify(ingestion).write(new Request(s, MetricCollection.points(points)));
}
 

private synchronized void findNextValues() {
    if (!wasClosed()) {
        try {
            long numOfValues = connection.readLong();
            if (numOfValues == LONG_END) {
                nextValues = null;
                readAllValuesFromConnection = true;
            } else if (numOfValues != LONG_ERROR) {
                byte[] keys = connection.readByteArray();
                byte[] compressedValues = connection.readByteArray();
                byte[] uncompressedValues = Snappy.uncompress(compressedValues);
                DataStream keyIS = new DataStream(keys);
                DataStream valueIS = new DataStream(uncompressedValues);
                List<KeyValue<T>> nextValuesList = new ArrayList<>();
                while (nextValuesList.size() < numOfValues) {
                    long key = keyIS.readLong();
                    int objectSize = DataStreamUtils.getObjectSize(valueIS, objectSerializer);
                    T value = objectSerializer.readValue(valueIS, objectSize);
                    nextValuesList.add(new KeyValue<>(key, value));
                }
                if (nextValuesList.isEmpty()) {
                    throw new RuntimeException("Received zero values! numOfValues=" + numOfValues);
                }
                nextValues = nextValuesList.iterator();
            } else {
                throw new RuntimeException("Unexpected response " + connection.readString());

            }
        } catch (Exception e) {
            remoteDataInterface.dropConnection(connection);
            throw new RuntimeException(e);
        }
    } else {
        nextValues = null;
    }
}
 

public static int compress(byte[] input, int inputOffset, int inputLength, byte[] output, int outputOffset) throws IOException {
    if (installed) {
        return Snappy.compress(input, inputOffset, inputLength, output, outputOffset);
    }
    System.arraycopy(input, inputOffset, output, outputOffset, inputLength);
    return inputLength;
}
 

private void writeCompressedBuf(ArrowBuf buf, OutputStream output) throws IOException {
  long rawLength = buf.readableBytes();
  for (long posn = 0; posn < rawLength; posn += RAW_CHUNK_SIZE_TO_COMPRESS) {
    /* we compress 32KB chunks at a time; the last chunk might be smaller than 32KB */
    int lengthToCompress = (int) Math.min(RAW_CHUNK_SIZE_TO_COMPRESS, rawLength - posn);

    /* allocate direct buffers to hold raw and compressed data */
    ByteBuffer rawDirectBuffer = buf.nioBuffer(posn, lengthToCompress);
    /* Since we don't know the exact size of compressed data, we can
     * allocate the compressed buffer of same size as raw data. However,
     * there could be cases where Snappy does not compress the data and the
     * compressed stream is of size larger (raw data + compression metadata)
     * than raw data. To handle these cases, we allocate compressed buffer
     * slightly larger than raw buffer. If we don't do this, Snappy.compress
     * will segfault.
     */
    final int maxCompressedLength = Snappy.maxCompressedLength(lengthToCompress);
    try (ArrowBuf cBuf = allocator.buffer(maxCompressedLength)) {
      ByteBuffer compressedDirectBuffer = cBuf.nioBuffer(0, maxCompressedLength);
      rawDirectBuffer.order(ByteOrder.LITTLE_ENDIAN);
      compressedDirectBuffer.order(ByteOrder.LITTLE_ENDIAN);

      /* compress */
      int compressedLength = Snappy.compress(rawDirectBuffer, compressedDirectBuffer);

      /* get compressed data into byte array for serializing to output stream */
      /* Use current thread buffer (safe to do since I/O operation is blocking) */
      final byte[] tmpBuffer = REUSABLE_LARGE_BUFFER.get();
      compressedDirectBuffer.get(tmpBuffer, 0, compressedLength);

      /* serialize the length of compressed data */
      output.write(getByteArrayFromLEInt(REUSABLE_SMALL_BUFFER.get(), compressedLength));
      /* serialize the compressed data */
      output.write(tmpBuffer, 0, compressedLength);
    }
  }
}
 

private static void serializeShortArray(final DataOutput out, final short[] val, boolean compress)
    throws IOException {
  if (compress && val.length > 250) {
    out.write(SHORT_ARRAY_C);
    byte[] b = Snappy.compress(val);
    LongPacker.packInt(out, b.length);
    out.write(b);
  } else {
    out.write(SHORT_ARRAY);
    LongPacker.packInt(out, val.length);
    for (short s : val) {
      out.writeShort(s);
    }
  }
}
 

/**
 * Create a new instance of the SnappyCompressor.
 * 
 * @throws IllegaltStateException when the Snappy native library is unavailable
 */
@SuppressFBWarnings(value="DMI_RANDOM_USED_ONLY_ONCE", justification="The native library is only used once so we only need to generate one random number")
public SnappyCompressor() {
  synchronized (defaultInstance) {
    if (!nativeLibraryLoaded) {
      try {
        SnappyUtils.setSnappySystemProperties(SnappyUtils.getOrCreateLogWriter());
        Snappy.getNativeLibraryVersion();
      } catch (SnappyError se) {
        throw new IllegalStateException(LocalizedStrings.SnappyCompressor_UNABLE_TO_LOAD_NATIVE_SNAPPY_LIBRARY.toLocalizedString(), se);
      }
      nativeLibraryLoaded = true;
    }
  }
}
 

/**
  * Fills specified buffer with uncompressed data. Returns actual number
  * of bytes of uncompressed data. A return value of 0 indicates that
  * {@link #needsInput()} should be called in order to determine if more
  * input data is required.
  *
  * @param buffer   Buffer for the compressed data
  * @param off Start offset of the data
  * @param len Size of the buffer
  * @return The actual number of bytes of uncompressed data.
  * @throws IOException if reading or decompression fails
  */
 @Override
 public synchronized int decompress(byte[] buffer, int off, int len) throws IOException {
   SnappyUtil.validateBuffer(buffer, off, len);
if (inputBuffer.position() == 0 && !outputBuffer.hasRemaining()) {
     return 0;
   }
   
   if (!outputBuffer.hasRemaining()) {
     inputBuffer.rewind();
     Preconditions.checkArgument(inputBuffer.position() == 0, "Invalid position of 0.");
     Preconditions.checkArgument(outputBuffer.position() == 0, "Invalid position of 0.");
     // There is compressed input, decompress it now.
     int decompressedSize = Snappy.uncompressedLength(inputBuffer);
     if (decompressedSize > outputBuffer.capacity()) {
       ByteBuffer oldBuffer = outputBuffer;
       outputBuffer = ByteBuffer.allocateDirect(decompressedSize);
       CleanUtil.cleanDirectBuffer(oldBuffer);
     }

     // Reset the previous outputBuffer (i.e. set position to 0)
     outputBuffer.clear();
     int size = Snappy.uncompress(inputBuffer, outputBuffer);
     outputBuffer.limit(size);
     // We've decompressed the entire input, reset the input now
     inputBuffer.clear();
     inputBuffer.limit(0);
     finished = true;
   }

   // Return compressed output up to 'len'
   int numBytes = Math.min(len, outputBuffer.remaining());
   outputBuffer.get(buffer, off, numBytes);
   return numBytes;	    
 }
 

public static byte[] compressFilter(byte[] datas) {
    if (Constants.COMPRESS_FILE) {
        try {
            datas = Snappy.compress(datas);
        } catch (IOException e) {
            log.error("Compress error!", e);
        }
    }
    return datas;
}
 

/**
 * Using snappy uncompress.
 * 
 * @param data
 * @return
 */
public static byte[] snappyUnCompress(byte[] data) {
	try {
		return Snappy.uncompress(data);
	} catch (IOException e) {
		throw new IllegalStateException(e);
	}
}
 

private void fetchBlock() throws IOException {
  int compressedSize = Util.readUnsignedVLQInt(input);
  input.read(compressedBuf, 0, compressedSize);
  int uncompressedSize = Snappy.uncompress(compressedBuf, 0, compressedSize, uncompressedBuf, 0);
  bufPos = 0;
  blockSize = uncompressedSize;

  curBlockStart = nextBlockStart;
  nextBlockStart = curBlockStart + Util.unsignedVLQSize(compressedSize) + compressedSize;
}
 

void reportSnappy2Agent(Address agentAddress, String msg, Map<String, String> metadata) {
    HttpPost httpPost = new HttpPost(buildRealAgentServerURL(agentAddress));
    httpPost.setHeader(HttpHeaders.CONTENT_TYPE, APPLICATION_OCTET_STREAM);
    httpPost.setHeader(HttpHeaders.CONTENT_ENCODING, SNAPPY);
    byte[] compressed = new byte[0];
    try {
        compressed = Snappy.compress(msg, Charset.forName(UTF_8));
    } catch (IOException e) {
        logger.info(">>WARNING: snappy compress report msg err:{}", e.getMessage());
        return;
    }
    httpPost.setEntity(new ByteArrayEntity(compressed));
    sendHttpDataSilently(httpPost, metadata);
}
 

@Test
public void shouldDecompressLargeContent() throws Exception {
    String text = "Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Aenean commodo ligula " +
        "eget dolor. Aenean massa. Cum sociis natoque penatibus et magnis dis parturient montes, " +
        "nascetur ridiculus mus. Donec quam felis, ultricies nec, pellentesque eu, pretium quis, sem. " +
        "Nulla consequat massa quis enim. Donec pede justo, fringilla vel, aliquet nec, vulputate " +
        "eget, arcu. In enim justo, rhoncus ut, imperdiet a, venenatis vitae, justo. Nullam dictum " +
        "felis eu pede mollis pretium. Integer tincidunt. Cras dapibus. Vivamus elementum semper " +
        "nisi. Aenean vulputate eleifend tellus. Aenean leo ligula, porttitor eu, consequat vitae, " +
        "eleifend ac, enim. Aliquam lorem ante, dapibus in, viverra quis, feugiat a, tellus. " +
        "Phasellus viverra nulla ut metus varius laoreet. Quisque rutrum. Aenean imperdiet. Etiam " +
        "ultricies nisi vel augue. Curabitur ullamcorper ultricies nisi. Nam eget dui. Etiam rhoncus." +
        " Maecenas tempus, tellus eget condimentum rhoncus, sem quam semper libero, sit amet " +
        "adipiscing sem neque sed ipsum. Nam quam nunc, blandit vel, luctus pulvinar, hendrerit id, " +
        "lorem. Maecenas nec odio et ante tincidunt tempus. Donec vitae sapien ut libero venenatis " +
        "faucibus. Nullam quis ante. Etiam sit amet orci eget eros faucibus tincidunt. Duis leo. " +
        "Sed fringilla mauris sit amet nibh. Donec sodales sagittis magna. Sed consequat, leo eget" +
        " bibendum sodales, augue velit cursus nunc,";

    // make sure decompression also works on large chunks....
    while (text.length() < Short.MAX_VALUE) {
        text += text;
    }

    channel.pipeline().addFirst(new SnappyFeatureHandler());

    ByteBuf content = Unpooled.wrappedBuffer(Snappy.compress(text.getBytes(CharsetUtil.UTF_8)));
    FullBinaryMemcacheResponse response = new DefaultFullBinaryMemcacheResponse(KEY, Unpooled.EMPTY_BUFFER,
        content);
    response.setDataType(KeyValueHandler.DATATYPE_SNAPPY);

    GetRequest requestMock = mock(GetRequest.class);
    requestQueue.add(requestMock);
    channel.writeInbound(response);

    GetResponse event = (GetResponse) eventSink.responseEvents().get(0).getMessage();
    assertEquals(text, event.content().toString(CHARSET));
    ReferenceCountUtil.release(event);
}
 

@Override
public ByteBuf compressInput(Object proto, RpcMethodInfo rpcMethodInfo) throws IOException {
    byte[] bytes = rpcMethodInfo.inputEncode(proto);
    int maxCompressedSize = Snappy.maxCompressedLength(bytes.length);
    byte[] compressedBytes = new byte[maxCompressedSize];
    int compressedLen = Snappy.compress(bytes, 0, bytes.length, compressedBytes, 0);
    return Unpooled.wrappedBuffer(compressedBytes, 0, compressedLen);
}
 

public Bytes compress(final Bytes data) {
  try {
    return Bytes.wrap(Snappy.compress(data.toArrayUnsafe()));
  } catch (IOException e) {
    throw new RuntimeException("Unable to compress data", e);
  }
}