net.jpountz.lz4.LZ4Compressor Java Examples

The following examples show how to use net.jpountz.lz4.LZ4Compressor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SerializationUtils.java    From exchange-core with Apache License 2.0 6 votes vote down vote up
public static long[] bytesToLongArrayLz4(final LZ4Compressor lz4Compressor, final NativeBytes<Void> bytes, final int padding) {
        int originalSize = (int) bytes.readRemaining();
//        log.debug("COMPRESS originalSize={}", originalSize);

        final ByteBuffer byteBuffer = ByteBuffer.allocate(originalSize);

        bytes.read(byteBuffer);

        byteBuffer.flip();

        final ByteBuffer byteBufferCompressed = ByteBuffer.allocate(4 + lz4Compressor.maxCompressedLength(originalSize));
        byteBufferCompressed.putInt(originalSize);// override with compressed length
        lz4Compressor.compress(byteBuffer, byteBufferCompressed);

        byteBufferCompressed.flip();

        int compressedBytesLen = byteBufferCompressed.remaining();

        return toLongsArray(
                byteBufferCompressed.array(),
                byteBufferCompressed.arrayOffset(),
                compressedBytesLen,
                padding);
    }
 
Example #2
Source File: CompressionUtil.java    From consulo with Apache License 2.0 6 votes vote down vote up
public static int writeCompressedWithoutOriginalBufferLength(@Nonnull DataOutput out, @Nonnull byte[] bytes, int length) throws IOException {
  long started = DUMP_COMPRESSION_STATS ? System.nanoTime() : 0;

  LZ4Compressor compressor = compressor();
  final byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
  int compressedSize = compressor.compress(bytes, 0, length, compressedOutputBuffer, 0);

  final long time = (DUMP_COMPRESSION_STATS ? System.nanoTime() : 0) - started;
  mySizeAfterCompression.addAndGet(compressedSize);
  mySizeBeforeCompression.addAndGet(length);
  int requests = myCompressionRequests.incrementAndGet();
  long l = myCompressionTime.addAndGet(time);

  if (DUMP_COMPRESSION_STATS && (requests & 0x1fff) == 0) {
    System.out.println("Compressed " + requests + " times, size:" + mySizeBeforeCompression + "->" + mySizeAfterCompression + " for " + (l / 1000000) + "ms");
  }

  DataInputOutputUtil.writeINT(out, compressedSize);
  out.write(compressedOutputBuffer, 0, compressedSize);

  return compressedSize;
}
 
Example #3
Source File: CompressionUtil.java    From consulo with Apache License 2.0 6 votes vote down vote up
public static int writeCompressed(@Nonnull DataOutput out, @Nonnull byte[] bytes, int start, int length) throws IOException {
  if (length > COMPRESSION_THRESHOLD) {
    LZ4Compressor compressor = compressor();

    byte[] compressedOutputBuffer = spareBufferLocal.getBuffer(compressor.maxCompressedLength(length));
    int compressedSize = compressor.compress(bytes, start, length, compressedOutputBuffer, 0);
    if (compressedSize < length) {
      DataInputOutputUtil.writeINT(out, -compressedSize);
      DataInputOutputUtil.writeINT(out, length - compressedSize);
      out.write(compressedOutputBuffer, 0, compressedSize);
      return compressedSize;
    }
  }
  DataInputOutputUtil.writeINT(out, length);
  out.write(bytes, start, length);
  return length;
}
 
Example #4
Source File: Compressor.java    From vespa with Apache License 2.0 6 votes vote down vote up
public long warmup(double seconds) {
    byte [] input = new byte[0x4000];
    new Random().nextBytes(input);
    long timeDone = System.nanoTime() + (long)(seconds*1000000000);
    long compressedBytes = 0;
    byte [] decompressed = new byte [input.length];
    LZ4FastDecompressor fastDecompressor = factory.fastDecompressor();
    LZ4SafeDecompressor safeDecompressor = factory.safeDecompressor();
    LZ4Compressor fastCompressor = factory.fastCompressor();
    LZ4Compressor highCompressor = factory.highCompressor();
    while (System.nanoTime() < timeDone) {
        byte [] compressedFast = fastCompressor.compress(input);
        byte [] compressedHigh = highCompressor.compress(input);
        fastDecompressor.decompress(compressedFast, decompressed);
        fastDecompressor.decompress(compressedHigh, decompressed);
        safeDecompressor.decompress(compressedFast, decompressed);
        safeDecompressor.decompress(compressedHigh, decompressed);
        compressedBytes += compressedFast.length + compressedHigh.length;
    }
    return compressedBytes;
}
 
Example #5
Source File: LZ4Codec.java    From redisson with Apache License 2.0 6 votes vote down vote up
@Override
public ByteBuf encode(Object in) throws IOException {
    ByteBuf bytes = null;
    try {
        LZ4Compressor compressor = factory.fastCompressor();
        bytes = innerCodec.getValueEncoder().encode(in);
        ByteBuffer srcBuf = bytes.internalNioBuffer(bytes.readerIndex(), bytes.readableBytes());
        
        int outMaxLength = compressor.maxCompressedLength(bytes.readableBytes());
        ByteBuf out = ByteBufAllocator.DEFAULT.buffer(outMaxLength + DECOMPRESSION_HEADER_SIZE);
        out.writeInt(bytes.readableBytes());
        ByteBuffer outBuf = out.internalNioBuffer(out.writerIndex(), out.writableBytes());
        int pos = outBuf.position();
        
        compressor.compress(srcBuf, outBuf);
        
        int compressedLength = outBuf.position() - pos;
        out.writerIndex(out.writerIndex() + compressedLength);
        return out;
    } finally {
        if (bytes != null) {
            bytes.release();
        }
    }
}
 
Example #6
Source File: CompressionUtil.java    From consulo with Apache License 2.0 5 votes vote down vote up
@Nonnull
public static Object compressStringRawBytes(@Nonnull CharSequence string) {
  int length = string.length();
  if (length < STRING_COMPRESSION_THRESHOLD) {
    if (string instanceof CharBuffer && ((CharBuffer)string).capacity() > STRING_COMPRESSION_THRESHOLD) {
      string = string.toString();   // shrink to size
    }
    return string;
  }
  try {
    BufferExposingByteArrayOutputStream bytes = new BufferExposingByteArrayOutputStream(length);
    @Nonnull DataOutput out = new DataOutputStream(bytes);

    for (int i = 0; i < length; i++) {
      char c = string.charAt(i);
      DataInputOutputUtil.writeINT(out, c);
    }

    LZ4Compressor compressor = compressor();
    int bytesWritten = bytes.size();
    ByteBuffer dest = ByteBuffer.wrap(spareBufferLocal.getBuffer(compressor.maxCompressedLength(bytesWritten) + 10));
    DataInputOutputUtil.writeINT(dest, length);
    DataInputOutputUtil.writeINT(dest, bytesWritten - length);
    compressor.compress(ByteBuffer.wrap(bytes.getInternalBuffer(), 0, bytesWritten), dest);

    return dest.position() < length * 2 ? Arrays.copyOf(dest.array(), dest.position()) : string;
  }
  catch (IOException e) {
    e.printStackTrace();
    return string;
  }
}
 
Example #7
Source File: LZ4CompressorTest.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
 
Example #8
Source File: Lz4CompressUtils.java    From hibernate4-memcached with Apache License 2.0 5 votes vote down vote up
public static byte[] compress(final byte[] src) {
    if (src == null) {
        throw new IllegalArgumentException("src must not be null.");
    }

    LZ4Compressor compressor = factory.fastCompressor();

    return compressor.compress(src);
}
 
Example #9
Source File: LZ4ObjectCache.java    From Arabesque with Apache License 2.0 5 votes vote down vote up
private void compressDataOutput() {
    if (!Configuration.get().isUseCompressedCaches() || !uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}
 
Example #10
Source File: ODAGStashLZ4Wrapper.java    From Arabesque with Apache License 2.0 5 votes vote down vote up
private void compress() {
    if (!uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}
 
Example #11
Source File: ODAGPartLZ4Wrapper.java    From Arabesque with Apache License 2.0 5 votes vote down vote up
private void compress() {
    if (!uncompressed) {
        return;
    }

    uncompressedSize = byteArrayOutputCache.getPos();
    LZ4Compressor lz4Compressor = lz4factory.fastCompressor();
    int maxCompressedLength = lz4Compressor.maxCompressedLength(uncompressedSize);
    ByteBuffer compressed = ByteBuffer.wrap(new byte[maxCompressedLength]);
    int compressedLength = lz4Compressor.compress(ByteBuffer.wrap(byteArrayOutputCache.getByteArray()), 0,
            uncompressedSize, compressed, 0, maxCompressedLength);
    byteArrayOutputCache = new ExtendedByteArrayDataOutput(compressed.array(), compressedLength);
    uncompressed = false;
}
 
Example #12
Source File: LZ4CompressorTest.java    From kylin with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
 
Example #13
Source File: ChannelLZ4Compressor.java    From datakernel with Apache License 2.0 5 votes vote down vote up
private static ByteBuf compressBlock(LZ4Compressor compressor, StreamingXXHash32 checksum, byte[] bytes, int off, int len) {
	checkArgument(len != 0);

	int compressionLevel = compressionLevel(max(len, MIN_BLOCK_SIZE));

	int outputBufMaxSize = HEADER_LENGTH + ((compressor == null) ? len : compressor.maxCompressedLength(len));
	ByteBuf outputBuf = ByteBufPool.allocate(outputBufMaxSize);
	outputBuf.put(MAGIC);

	byte[] outputBytes = outputBuf.array();

	checksum.reset();
	checksum.update(bytes, off, len);
	int check = checksum.getValue();

	int compressedLength = len;
	if (compressor != null) {
		compressedLength = compressor.compress(bytes, off, len, outputBytes, HEADER_LENGTH);
	}

	int compressMethod;
	if (compressor == null || compressedLength >= len) {
		compressMethod = COMPRESSION_METHOD_RAW;
		compressedLength = len;
		System.arraycopy(bytes, off, outputBytes, HEADER_LENGTH, len);
	} else {
		compressMethod = COMPRESSION_METHOD_LZ4;
	}

	outputBytes[MAGIC_LENGTH] = (byte) (compressMethod | compressionLevel);
	writeIntLE(compressedLength, outputBytes, MAGIC_LENGTH + 1);
	writeIntLE(len, outputBytes, MAGIC_LENGTH + 5);
	writeIntLE(check, outputBytes, MAGIC_LENGTH + 9);

	outputBuf.tail(HEADER_LENGTH + compressedLength);

	return outputBuf;
}
 
Example #14
Source File: CompressionProcessorImpl.java    From ignite with Apache License 2.0 5 votes vote down vote up
/**
 * @param compactPage Compacted page.
 * @param compactSize Compacted page size.
 * @param compressLevel Compression level.
 * @return Compressed page.
 */
private ByteBuffer compressPageLz4(ByteBuffer compactPage, int compactSize, int compressLevel) {
    LZ4Compressor compressor = Lz4.getCompressor(compressLevel);

    ByteBuffer compressedPage = compressBuf.get();

    copyPageHeader(compactPage, compressedPage, compactSize);
    compressor.compress(compactPage, compressedPage);

    compactPage.flip();
    compressedPage.flip();

    return compressedPage;
}
 
Example #15
Source File: Lz4Compress.java    From compress with MIT License 5 votes vote down vote up
@Override
public byte[] compress(byte[] data) throws IOException {
	LZ4Factory factory = LZ4Factory.fastestInstance();
	ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
	LZ4Compressor compressor = factory.fastCompressor();
	LZ4BlockOutputStream compressedOutput = new LZ4BlockOutputStream(byteOutput, 2048, compressor);
	compressedOutput.write(data);
	compressedOutput.close();
	
	return byteOutput.toByteArray();
}
 
Example #16
Source File: CompressedBuffedReaderTest.java    From ClickHouse-Native-JDBC with Apache License 2.0 5 votes vote down vote up
private byte[] compressedData(byte[] bytes) {
    LZ4Compressor lz4Compressor = LZ4Factory.safeInstance().fastCompressor();
    byte[] compressData = lz4Compressor.compress(bytes);
    byte[] data = new byte[compressData.length + 9 + 16];

    data[16] = (byte) (0x82 & 0xFF);
    System.arraycopy(compressData, 0, data, 9 + 16, compressData.length);
    System.arraycopy(littleEndian(compressData.length + 9), 0, data, 17, 4);
    System.arraycopy(littleEndian(bytes.length), 0, data, 21, 4);

    return data;
}
 
Example #17
Source File: ChannelLZ4Compressor.java    From datakernel with Apache License 2.0 4 votes vote down vote up
public static ChannelLZ4Compressor create(LZ4Compressor compressor) {
	return new ChannelLZ4Compressor(compressor);
}
 
Example #18
Source File: ChannelLZ4Compressor.java    From datakernel with Apache License 2.0 4 votes vote down vote up
private ChannelLZ4Compressor(LZ4Compressor compressor) {
	this.compressor = compressor;
}
 
Example #19
Source File: CompressionProcessorImpl.java    From ignite with Apache License 2.0 4 votes vote down vote up
/**
 * @param level Compression level.
 * @return Compressor.
 */
static LZ4Compressor getCompressor(int level) {
    assert level >= 0 && level <= 17 : level;
    return level == 0 ? fastCompressor : factory.highCompressor(level);
}
 
Example #20
Source File: CloverDataStream.java    From CloverETL-Engine with GNU Lesser General Public License v2.1 4 votes vote down vote up
private CompressorLZ4(LZ4Compressor compressor) {
	this.compressor = compressor;
}
 
Example #21
Source File: Compressor.java    From vespa with Apache License 2.0 4 votes vote down vote up
private LZ4Compressor getCompressor() {
    return level < 7 ? factory.fastCompressor() : factory.highCompressor();
}
 
Example #22
Source File: NormalSketch.java    From vespa with Apache License 2.0 4 votes vote down vote up
/**
 * Serializes the Sketch.
 *
 * Serialization format
 * ==================
 * Original size:     4 bytes
 * Compressed size:   4 bytes
 * Compressed data:   N * 1 bytes
 *
 * Invariant:
 *      compressed size &lt;= original size
 *
 * Special case:
 *      compressed size == original size =&gt; data is uncompressed
 *
 * @param buf Serializer
 */
@Override
protected void onSerialize(Serializer buf) {
    super.onSerialize(buf);
    buf.putInt(null, data.length);
    try {
        LZ4Compressor c = lz4Factory.highCompressor();
        byte[] compressedData = new byte[data.length];
        int compressedSize = c.compress(data, compressedData);
        serializeDataArray(compressedData, compressedSize, buf);
    } catch (LZ4Exception e) {
        // LZ4Compressor.compress will throw this exception if it is unable to compress
        // into compressedData (when compressed size >= original size)
        serializeDataArray(data, data.length, buf);
    }
}
 
Example #23
Source File: CompressionUtil.java    From consulo with Apache License 2.0 4 votes vote down vote up
private static LZ4Compressor compressor() {
  return LZ4Factory.fastestJavaInstance().fastCompressor();
}