net.jpountz.lz4.LZ4FastDecompressor Java Examples

@Override
public byte[] uncompress(byte[] data) throws IOException {
	LZ4Factory factory = LZ4Factory.fastestInstance();
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	LZ4FastDecompressor decompresser = factory.fastDecompressor();
	LZ4BlockInputStream lzis = new LZ4BlockInputStream(new ByteArrayInputStream(data), decompresser);
	
	int count;
	byte[] buffer = new byte[2048];
	while ((count = lzis.read(buffer)) != -1) {
		baos.write(buffer, 0, count);
	}
	lzis.close();
	
	return baos.toByteArray();
}
 

@Override
protected void onDeserialize(Deserializer buf) {
    super.onDeserialize(buf);
    int length = buf.getInt(null);
    int compressedLength = buf.getInt(null);
    Preconditions.checkState(length == data.length,
            "Size of serialized sketch does not match expected value. Expected %s, actual %s.", data.length, length);

    if (length == compressedLength) {
        deserializeDataArray(data, length, buf);
    } else {
        LZ4FastDecompressor c = lz4Factory.fastDecompressor();
        byte[] compressedData = buf.getBytes(null, compressedLength);
        c.decompress(compressedData, data);
    }
}
 

public long warmup(double seconds) {
    byte [] input = new byte[0x4000];
    new Random().nextBytes(input);
    long timeDone = System.nanoTime() + (long)(seconds*1000000000);
    long compressedBytes = 0;
    byte [] decompressed = new byte [input.length];
    LZ4FastDecompressor fastDecompressor = factory.fastDecompressor();
    LZ4SafeDecompressor safeDecompressor = factory.safeDecompressor();
    LZ4Compressor fastCompressor = factory.fastCompressor();
    LZ4Compressor highCompressor = factory.highCompressor();
    while (System.nanoTime() < timeDone) {
        byte [] compressedFast = fastCompressor.compress(input);
        byte [] compressedHigh = highCompressor.compress(input);
        fastDecompressor.decompress(compressedFast, decompressed);
        fastDecompressor.decompress(compressedHigh, decompressed);
        safeDecompressor.decompress(compressedFast, decompressed);
        safeDecompressor.decompress(compressedHigh, decompressed);
        compressedBytes += compressedFast.length + compressedHigh.length;
    }
    return compressedBytes;
}
 

private void print(Request request) {
    Values ret = request.returnValues();
    CompressionType type = CompressionType.valueOf(ret.get(0).asInt8());
    int uncompressedSize = ret.get(1).asInt32();
    byte [] blob = ret.get(2).asData();
    if (type == CompressionType.LZ4) {
        LZ4FastDecompressor decompressor = lz4Factory.fastDecompressor();
        byte [] uncompressed = new byte [uncompressedSize];
        int compressedLength = decompressor.decompress(blob, 0, uncompressed, 0, uncompressedSize);
        if (compressedLength != blob.length) {
            throw new DeserializationException("LZ4 decompression failed. compressed size does not match. Expected " + blob.length + ". Got " + compressedLength);
        }
        blob = uncompressed;
    }
    Slime slime = BinaryFormat.decode(blob);
    try {
        new JsonFormat(true).encode(System.out, slime);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}
 

/**
 * When the exact decompressed size is known, use this method to decompress. It's faster.
 *
 * @see net.jpountz.lz4.LZ4FastDecompressor
 */
public static byte[] decompressFast(byte[] src, int srcOffset, int exactDecompressedSize) {
    if (src == null) {
        throw new IllegalArgumentException("src must not be null.");
    }

    if (srcOffset < 0) {
        throw new IllegalArgumentException("srcOffset must equal to or larger than 0 but " + srcOffset);
    }

    if (exactDecompressedSize < 0) {
        throw new IllegalArgumentException("exactDecompressedSize must equal to or larger than 0 but " + exactDecompressedSize);
    }

    LZ4FastDecompressor decompressor = factory.fastDecompressor();

    return decompressor.decompress(src, srcOffset, exactDecompressedSize);
}
 

private synchronized void decompress() {
    if (uncompressed) {
        return;
    }

    //LOG.info("Big no no");

    //long start = System.currentTimeMillis();
    LZ4FastDecompressor decompressor = lz4factory.fastDecompressor();

    ByteBuffer dest = ByteBuffer.allocate(uncompressedSize);
    ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos());

    decompressor.decompress(src, 0, dest, 0, uncompressedSize);

    byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize);
    uncompressed = true;
}
 

private void decompressDataInput() {
    if (!Configuration.get().isUseCompressedCaches() || uncompressed) {
        return;
    }

    //LOG.info("Big no no");

    LZ4FastDecompressor decompressor = lz4factory.fastDecompressor();

    ByteBuffer dest = ByteBuffer.allocate(uncompressedSize);
    ByteBuffer src = ByteBuffer.wrap(byteArrayOutputCache.getByteArray(), 0, byteArrayOutputCache.getPos());

    decompressor.decompress(src, 0, dest, 0, uncompressedSize);

    byteArrayOutputCache = new ExtendedByteArrayDataOutput(dest.array(), uncompressedSize);
    uncompressed = true;
}
 

public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
 

private byte[] readNextBlock() throws IOException {
    int read = stream.read();
    if (read < 0) return null;

    byte[] checksum = new byte[16];
    checksum[0] = (byte)read;
    // checksum - 16 bytes.
    dataWrapper.readFully(checksum, 1, 15);
    ClickHouseBlockChecksum expected = ClickHouseBlockChecksum.fromBytes(checksum);
    // header:
    // 1 byte - 0x82 (shows this is LZ4)
    int magic = dataWrapper.readUnsignedByte();
    if (magic != MAGIC) throw new IOException("Magic is not correct: " + magic);
    // 4 bytes - size of the compressed data including 9 bytes of the header
    int compressedSizeWithHeader = dataWrapper.readInt();
    // 4 bytes - size of uncompressed data
    int uncompressedSize = dataWrapper.readInt();
    int compressedSize = compressedSizeWithHeader - 9; //header
    byte[] block = new byte[compressedSize];
    // compressed data: compressed_size - 9 байт.
    dataWrapper.readFully(block);

    ClickHouseBlockChecksum real = ClickHouseBlockChecksum.calculateForBlock((byte)magic, compressedSizeWithHeader, uncompressedSize, block, compressedSize);
    if (!real.equals(expected)) {
        throw new IllegalArgumentException("Checksum doesn't match: corrupted data.");
    }

    byte[] decompressed = new byte[uncompressedSize];
    LZ4FastDecompressor decompressor = factory.fastDecompressor();
    decompressor.decompress(block, 0, decompressed, 0, uncompressedSize);
    return decompressed;
}
 

public String getDataTypePriv() throws IOException {
	RandomAccessFile fin = new RandomAccessFile(sinfile,"r");
	byte[] res=null;
	byte[] rescomp=null;
	if (dataHeader.mmcfLen>0) {
		Object block = dataBlocks.firstElement();
		if (block instanceof AddrBlock) {
			res = new byte[(int)((AddrBlock)block).dataLen];
			fin.seek(getDataOffset()+((AddrBlock)block).dataOffset);
			fin.read(res);
			fin.close();
		}
		else {
			rescomp = new byte[(int)((LZ4ABlock)block).compDataLen];
			fin.seek(getDataOffset()+((LZ4ABlock)block).dataOffset);
			fin.read(rescomp);
			fin.close();
			LZ4Factory factory = LZ4Factory.fastestInstance();
			LZ4FastDecompressor decomp = factory.fastDecompressor();
			res = decomp.decompress(rescomp, (int)((LZ4ABlock)block).uncompDataLen);
		}
	}
	else {
		res = new byte[blocks.blocks[0].length];
		fin.seek(getDataOffset());
		fin.read(res);
		fin.close();
	}
	return getDataTypePriv(res);
}
 

private static ByteBuf decompress(LZ4FastDecompressor decompressor, StreamingXXHash32 checksum, Header header,
		byte[] bytes, int off) throws ParseException {
	ByteBuf outputBuf = ByteBufPool.allocate(header.originalLen);
	outputBuf.tail(header.originalLen);
	switch (header.compressionMethod) {
		case COMPRESSION_METHOD_RAW:
			System.arraycopy(bytes, off, outputBuf.array(), 0, header.originalLen);
			break;
		case COMPRESSION_METHOD_LZ4:
			try {
				int compressedLen2 = decompressor.decompress(bytes, off, outputBuf.array(), 0, header.originalLen);
				if (header.compressedLen != compressedLen2) {
					throw STREAM_IS_CORRUPTED;
				}
			} catch (LZ4Exception e) {
				throw new ParseException(ChannelLZ4Decompressor.class, "Stream is corrupted", e);
			}
			break;
		default:
			throw STREAM_IS_CORRUPTED;
	}
	checksum.reset();
	checksum.update(outputBuf.array(), 0, header.originalLen);
	if (checksum.getValue() != header.check) {
		throw STREAM_IS_CORRUPTED;
	}
	return outputBuf;
}
 

public static void main(String[] args) throws Exception {

        if(args.length == 0){
            System.out.println("args[0] must be data file path");
            return;
        }
        LZ4Factory factory = LZ4Factory.fastestInstance();

        byte[] data = Files.toByteArray(new File(args[0]));
        final int decompressedLength = data.length;

        // compress data
        LZ4Compressor compressor = factory.fastCompressor();
        long start = System.currentTimeMillis();
        int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
        byte[] compressed = new byte[maxCompressedLength];
        int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
        System.out.println("compress take:" + (System.currentTimeMillis() - start));
        System.out.println(compressedLength);

        // decompress data
        // - method 1: when the decompressed length is known
        LZ4FastDecompressor decompressor = factory.fastDecompressor();
        start = System.currentTimeMillis();
        byte[] restored = new byte[decompressedLength];
        int compressedLength2 = decompressor.decompress(compressed, 0, restored, 0, decompressedLength);
        System.out.println("decompress take:" + (System.currentTimeMillis() - start));
        System.out.println(decompressedLength);
        // compressedLength == compressedLength2

        // - method 2: when the compressed length is known (a little slower)
        // the destination buffer needs to be over-sized
        LZ4SafeDecompressor decompressor2 = factory.safeDecompressor();
        int decompressedLength2 = decompressor2.decompress(compressed, 0, compressedLength, restored, 0);
    }
 

protected static LZ4FastDecompressor decompressor() {
  return LZ4Factory.fastestJavaInstance().fastDecompressor();
}
 

private void read() throws IOException
{
    this.nso0 = new NSO0Header(this.fileReader, 0x0);
    
    LZ4Factory factory = LZ4Factory.fastestInstance();
    LZ4FastDecompressor decompressor = factory.fastDecompressor();
    
    NSO0SectionHeader textHeader = this.nso0.getSectionHeader(NXOSectionType.TEXT);
    NSO0SectionHeader rodataHeader = this.nso0.getSectionHeader(NXOSectionType.RODATA);
    NSO0SectionHeader dataHeader = this.nso0.getSectionHeader(NXOSectionType.DATA);
    
    int textOffset = textHeader.getMemoryOffset();
    int rodataOffset = rodataHeader.getMemoryOffset();
    int dataOffset = dataHeader.getMemoryOffset();
    int textSize = textHeader.getDecompressedSize();
    int rodataSize = rodataHeader.getDecompressedSize();
    int dataSize = dataHeader.getDecompressedSize();
    
    // The data section is last, so we use its offset + decompressed size
    byte[] full = new byte[dataOffset + dataSize];
    byte[] decompressedText;
    byte[] decompressedRodata;
    byte[] decompressedData;
    
    if (this.nso0.isSectionCompressed(NXOSectionType.TEXT))
    {
        byte[] compressedText = this.fileProvider.readBytes(this.nso0.getSectionFileOffset(NXOSectionType.TEXT), this.nso0.getCompressedSectionSize(NXOSectionType.TEXT));
        decompressedText = new byte[textSize];
        decompressor.decompress(compressedText, decompressedText);
    }
    else
    {
        decompressedText = this.fileProvider.readBytes(this.nso0.getSectionFileOffset(NXOSectionType.TEXT), textSize);
    }
    
    System.arraycopy(decompressedText, 0, full, textOffset, textSize);
    
    if (this.nso0.isSectionCompressed(NXOSectionType.RODATA))
    {
        byte[] compressedRodata = this.fileProvider.readBytes(this.nso0.getSectionFileOffset(NXOSectionType.RODATA), this.nso0.getCompressedSectionSize(NXOSectionType.RODATA));
        decompressedRodata = new byte[rodataSize];
        decompressor.decompress(compressedRodata, decompressedRodata);
    }
    else
    {
        decompressedRodata = this.fileProvider.readBytes(this.nso0.getSectionFileOffset(NXOSectionType.RODATA), rodataSize);
    }
    
    System.arraycopy(decompressedRodata, 0, full, rodataOffset, rodataSize);
    
    if (this.nso0.isSectionCompressed(NXOSectionType.DATA))
    {
        byte[] compressedData = this.fileProvider.readBytes(this.nso0.getSectionFileOffset(NXOSectionType.DATA), this.nso0.getCompressedSectionSize(NXOSectionType.DATA));
        decompressedData = new byte[dataSize];
        decompressor.decompress(compressedData, decompressedData);
    }
    else
    {
        decompressedData = this.fileProvider.readBytes(this.nso0.getSectionFileOffset(NXOSectionType.DATA), dataSize);
    }
    
    System.arraycopy(decompressedData, 0, full, dataOffset, dataSize);
    this.memoryProvider = new ByteArrayProvider(full);
    
    this.sections = new NXOSection[3];
    this.sections[NXOSectionType.TEXT.ordinal()] = new NXOSection(NXOSectionType.TEXT, textOffset, textSize);
    this.sections[NXOSectionType.RODATA.ordinal()] = new NXOSection(NXOSectionType.RODATA, rodataOffset, rodataSize);
    this.sections[NXOSectionType.DATA.ordinal()] = new NXOSection(NXOSectionType.DATA, dataOffset, dataSize);
}
 

private void receiveMessages() throws IOException
{
    // handshake (true) endpoint versions
    DataOutputStream out = new DataOutputStream(socket.getOutputStream());
    out.writeInt(MessagingService.current_version);
    out.flush();
    DataInputStream in = new DataInputStream(socket.getInputStream());
    int maxVersion = in.readInt();

    from = CompactEndpointSerializationHelper.deserialize(in);
    // record the (true) version of the endpoint
    MessagingService.instance().setVersion(from, maxVersion);
    logger.debug("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from));

    if (compressed)
    {
        logger.debug("Upgrading incoming connection to be compressed");
        if (version < MessagingService.VERSION_21)
        {
            in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
        }
        else
        {
            LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
            Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
            in = new DataInputStream(new LZ4BlockInputStream(socket.getInputStream(),
                                                             decompressor,
                                                             checksum));
        }
    }
    else
    {
        in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), BUFFER_SIZE));
    }

    if (version > MessagingService.current_version)
    {
        // save the endpoint so gossip will reconnect to it
        Gossiper.instance.addSavedEndpoint(from);
        logger.info("Received messages from newer protocol version {}. Ignoring", version);
        return;
    }
    // outbound side will reconnect if necessary to upgrade version

    while (true)
    {
        MessagingService.validateMagic(in.readInt());
        receiveMessage(in, version);
    }
}
 

public static ChannelLZ4Decompressor create(LZ4FastDecompressor decompressor, XXHashFactory xxHashFactory) {
	return new ChannelLZ4Decompressor(decompressor, xxHashFactory.newStreamingHash32(DEFAULT_SEED));
}
 

private ChannelLZ4Decompressor(LZ4FastDecompressor decompressor, StreamingXXHash32 checksum) {
	this.decompressor = decompressor;
	this.checksum = checksum;
}
 

public static Wire longsLz4ToWire(long[] dataArray, int longsTransfered) {

//        log.debug("long dataArray.len={} longsTransfered={}", dataArray.length, longsTransfered);

        final ByteBuffer byteBuffer = ByteBuffer.allocate(longsTransfered * 8);
        byteBuffer.asLongBuffer().put(dataArray, 0, longsTransfered);

        final int originalSizeBytes = byteBuffer.getInt();

        final ByteBuffer uncompressedByteBuffer = ByteBuffer.allocate(originalSizeBytes);

        final LZ4FastDecompressor lz4FastDecompressor = LZ4Factory.fastestInstance().fastDecompressor();

        lz4FastDecompressor.decompress(byteBuffer, byteBuffer.position(), uncompressedByteBuffer, uncompressedByteBuffer.position(), originalSizeBytes);

        final Bytes<ByteBuffer> bytes = Bytes.wrapForRead(uncompressedByteBuffer);

        return WireType.RAW.apply(bytes);
    }