net.jpountz.xxhash.XXHashFactory Java Examples

private Long computeHash(File file) throws IOException {
    XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
    if (file.isDirectory()) {
        return Files.walk(file.toPath(), 100).map(path -> {
            try {
                log.log(Level.FINE, "Calculating hash for '" + path + "'");
                return hash(path.toFile(), hasher);
            } catch (IOException e) {
                log.log(Level.WARNING, "Failed getting hash from '" + path + "'");
                return 0;
            }
        }).mapToLong(Number::longValue).sum();
    } else {
        return hash(file, hasher);
    }
}
 

Session(File downloadDirectory, File tmpDirectory, int sessionId, FileReference reference,
        FileReferenceData.Type fileType, String fileName, long fileSize)
{
    this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0);
    this.sessionId = sessionId;
    this.reference = reference;
    this.fileType = fileType;
    this.fileName = fileName;
    this.fileSize = fileSize;
    currentFileSize = 0;
    currentPartId = 0;
    currentHash = 0;
    fileReferenceDir = new File(downloadDirectory, reference.value());
    this.tmpDir = tmpDirectory;

    try {
        inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile();
    } catch (IOException e) {
        String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDirectory.toPath() + "': ";
        log.log(Level.SEVERE, msg + e.getMessage(), e);
        throw new RuntimeException(msg, e);
    }
}
 

void receive(FileReference reference, String filename, byte[] content) {

            log.log(Level.INFO, "Preparing receive call for " + reference.value() + " and file " + filename);

            XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
            Request fileBlob = new Request("filedistribution.receiveFile");

            log.log(Level.INFO, "Calling " + fileBlob.methodName() + " with target " + target);

            fileBlob.parameters().add(new StringValue(reference.value()));
            fileBlob.parameters().add(new StringValue(filename));
            fileBlob.parameters().add(new DataValue(content));
            fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0)));
            fileBlob.parameters().add(new Int32Value(0));
            fileBlob.parameters().add(new StringValue("OK"));
            log.log(Level.INFO, "Doing invokeSync");
            target.invokeSync(fileBlob, 5);
            log.log(Level.INFO, "Done with invokeSync");
        }
 

private void receiveFile(FileDownloader fileDownloader, FileReference fileReference, String filename,
                         FileReferenceData.Type type, byte[] content) {
    XXHash64 hasher = XXHashFactory.fastestInstance().hash64();
    FileReceiver.Session session =
            new FileReceiver.Session(downloadDir, tempDir, 1, fileReference, type, filename, content.length);
    session.addPart(0, content);
    File file = session.close(hasher.hash(ByteBuffer.wrap(content), 0));
    fileDownloader.fileReferenceDownloader().completedDownloading(fileReference, file);
}
 

/**
 * Creates a hex number from the input.
 * From either a `string`, `Uint8Array` or a `Buffer` input, create the xxhash and return the result as a hex number
 * **example**
 * <p>
 * ```java
 * xxhash64AsValue("abcd", 0xabcd)); // => e29f70f8b8c96df7
 * ```
 */
//export default function xxhash64AsValue (data: Buffer | Uint8Array | string, seed: number): number {
public static long xxhash64AsValue(byte[] data, long seed) {

    XXHashFactory factory = XXHashFactory.fastestInstance();
    ByteArrayInputStream in = new ByteArrayInputStream(data);
    // value you want, but always the same
    StreamingXXHash64 xxHash64 = factory.newStreamingHash64(seed);

    byte[] buf = new byte[16]; // for real-world usage, use a larger buffer, like 8192 bytes
    for (; ; ) {
        int read = 0;
        try {
            read = in.read(buf);
        } catch (IOException e) {
            e.printStackTrace();
        }
        if (read == -1) {
            break;
        }
        xxHash64.update(buf, 0, read);
    }
    long hash = xxHash64.getValue();
    //System.out.println("value :" + Arrays.toString(data));
    //System.out.println("seed :" + seed);
    //System.out.println("hash :" + Long.toHexString(hash));
    return hash;
}
 

public static final void register(final Map<String, Hasher> hashers) {
  hashers.put(Jp32Hasher.XXH32_JNI,
          new Jp32Hasher(XXHashFactory.nativeInstance().hash32()));
  hashers.put(Jp32Hasher.XXH32_UNSAFE,
          new Jp32Hasher(XXHashFactory.unsafeInstance().hash32()));
  hashers.put(Jp32Hasher.XXH32_SAFE,
          new Jp32Hasher(XXHashFactory.safeInstance().hash32()));
}
 

@Activate
void activate(Map<String, Object> configuration) throws VirtualFilesystemException {
    SimpleVirtualFilesystemConfig conf = Configurable.createConfigurable(SimpleVirtualFilesystemConfig.class,
            configuration);
    try {
        this.fsManager = VFS.getManager();
        File rootDirectory = new File(conf.defaultRootDirectory());
        if (!rootDirectory.exists()) {
            rootDirectory.mkdirs();
        }
        ((DefaultFileSystemManager) this.fsManager).setBaseFile(rootDirectory);
    } catch (FileSystemException e) {
        throw new VirtualFilesystemException("Issue initializing virtual file system.", e);
    }
    // Set of queues.
    tempFiles = new ArrayBlockingQueue<>(conf.maxNumberOfTempFiles());

    // Schedule our temp file cleanup service.
    this.scheduledExecutorService = Executors.newScheduledThreadPool(1);
    this.scheduledExecutorService.scheduleAtFixedRate(new CleanTempFilesRunnable(LOGGER, tempFiles),
            conf.secondsBetweenTempCleanup(), conf.secondsBetweenTempCleanup(), TimeUnit.SECONDS);
    LOGGER.debug("Configured scheduled cleanup of temp files to run every {} seconds",
            conf.secondsBetweenTempCleanup());

    // Set default temp url template
    this.baseTempUrlTemplate = conf.defaultTemporaryDirectory() != null ? conf.defaultTemporaryDirectory() :
            ("file://" + System.getProperty("java.io.tmpdir"));
    LOGGER.debug("Going to use {} for our base temp directory template", this.baseTempUrlTemplate);

    // Initialize HashFactory
    this.hashFactory = XXHashFactory.fastestInstance();
}
 

@Test
public void calcHashCode() {
    int id = 10;
    var hashCodeCalculator = new XXHash32BinaryHashCodeCalculator(XXHashFactory.safeInstance().hash32(), id);
    var bytes = new byte[] {0x00, 0x01, 0x02, 0x03};
    var ans = hashCodeCalculator.calcHashCode(bytes);
    assertEquals(1256378755, ans);
    assertEquals(id, hashCodeCalculator.getId());
}
 

public static final void register(final Map<String, Hasher> hashers) {
  hashers.put(Jp64Hasher.XXH64_JNI,
          new Jp64Hasher(XXHashFactory.nativeInstance().hash64()));
  hashers.put(Jp64Hasher.XXH64_UNSAFE,
          new Jp64Hasher(XXHashFactory.unsafeInstance().hash64()));
  hashers.put(Jp64Hasher.XXH64_SAFE,
          new Jp64Hasher(XXHashFactory.safeInstance().hash64()));
}
 

/**
 * Create a new {@link InputStream} that will decompress data using the LZ4 algorithm.
 *
 * @param in The stream to decompress
 * @throws IOException
 */
public KafkaLZ4BlockInputStream(InputStream in, boolean ignoreFlagDescriptorChecksum) throws IOException {
    super(in);
    this.ignoreFlagDescriptorChecksum = ignoreFlagDescriptorChecksum;
    decompressor = LZ4Factory.fastestInstance().safeDecompressor();
    checksum = XXHashFactory.fastestInstance().hash32();
    readHeader();
    maxBlockSize = bd.getBlockMaximumSize();
    buffer = new byte[maxBlockSize];
    compressedBuffer = new byte[maxBlockSize];
    bufferOffset = 0;
    bufferSize = 0;
    finished = false;

}
 

/**
 * Create a new {@link OutputStream} that will compress data using the LZ4 algorithm.
 *
 * @param out The output stream to compress
 * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other values will generate an exception
 * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for every block of data
 * @throws IOException
 */
public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum) throws IOException {
    super(out);
    compressor = LZ4Factory.fastestInstance().fastCompressor();
    checksum = XXHashFactory.fastestInstance().hash32();
    bd = new BD(blockSize);
    flg = new FLG(blockChecksum);
    bufferOffset = 0;
    maxBlockSize = bd.getBlockMaximumSize();
    buffer = new byte[maxBlockSize];
    compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)];
    finished = false;
    writeHeader();
}
 

/**
 * Create a new {@link java.io.InputStream} that will decompress data using the LZ4 algorithm.
 *
 * @param in The stream to decompress
 * @throws java.io.IOException
 */
public KafkaLZ4BlockInputStream(InputStream in, boolean ignoreFlagDescriptorChecksum) throws IOException {
    super(in);
    this.ignoreFlagDescriptorChecksum = ignoreFlagDescriptorChecksum;
    decompressor = LZ4Factory.fastestInstance().safeDecompressor();
    checksum = XXHashFactory.fastestInstance().hash32();
    readHeader();
    maxBlockSize = bd.getBlockMaximumSize();
    buffer = new byte[maxBlockSize];
    compressedBuffer = new byte[maxBlockSize];
    bufferOffset = 0;
    bufferSize = 0;
    finished = false;

}
 

/**
 * Create a new {@link java.io.OutputStream} that will compress data using the LZ4 algorithm.
 *
 * @param out The output stream to compress
 * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other values will generate an exception
 * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for every block of data
 * @throws java.io.IOException
 */
public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum) throws IOException {
    super(out);
    compressor = LZ4Factory.fastestInstance().fastCompressor();
    checksum = XXHashFactory.fastestInstance().hash32();
    bd = new BD(blockSize);
    flg = new FLG(blockChecksum);
    bufferOffset = 0;
    maxBlockSize = bd.getBlockMaximumSize();
    buffer = new byte[maxBlockSize];
    compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)];
    finished = false;
    writeHeader();
}
 

/**
 * Create a new {@link InputStream} that will decompress data using the LZ4 algorithm.
 *
 * @param in The stream to decompress
 * @throws IOException
 */
public KafkaLZ4BlockInputStream(InputStream in, boolean ignoreFlagDescriptorChecksum) throws IOException {
    super(in);
    this.ignoreFlagDescriptorChecksum = ignoreFlagDescriptorChecksum;
    decompressor = LZ4Factory.fastestInstance().safeDecompressor();
    checksum = XXHashFactory.fastestInstance().hash32();
    readHeader();
    maxBlockSize = bd.getBlockMaximumSize();
    buffer = new byte[maxBlockSize];
    compressedBuffer = new byte[maxBlockSize];
    bufferOffset = 0;
    bufferSize = 0;
    finished = false;

}
 

/**
 * Create a new {@link OutputStream} that will compress data using the LZ4 algorithm.
 *
 * @param out The output stream to compress
 * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other values will generate an exception
 * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for every block of data
 * @throws IOException
 */
public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum) throws IOException {
    super(out);
    compressor = LZ4Factory.fastestInstance().fastCompressor();
    checksum = XXHashFactory.fastestInstance().hash32();
    bd = new BD(blockSize);
    flg = new FLG(blockChecksum);
    bufferOffset = 0;
    maxBlockSize = bd.getBlockMaximumSize();
    buffer = new byte[maxBlockSize];
    compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)];
    finished = false;
    writeHeader();
}
 

private Lz4FrameEncoder newEncoder(int blockSize, int maxEncodeSize) {
    Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum();
    Lz4FrameEncoder encoder = new Lz4FrameEncoder(LZ4Factory.fastestInstance(), true,
                                                  blockSize,
                                                  checksum,
                                                  maxEncodeSize);
    encoder.handlerAdded(ctx);
    return encoder;
}
 

private static String uriToRelativeFile(String uri) {
    String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0);
    if (uri.endsWith(".json")) {
        relative += ".json";
    } else if (uri.endsWith(".json.lz4")) {
        relative += ".json.lz4";
    } else if (uri.endsWith(".lz4")) {
        relative += ".lz4";
    }
    return relative;
}
 

public static ChannelLZ4Decompressor create() {
	return create(
			LZ4Factory.fastestInstance().fastDecompressor(),
			XXHashFactory.fastestInstance());
}
 

public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content) {
    this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0));
}
 

public static ChannelLZ4Decompressor create(LZ4FastDecompressor decompressor, XXHashFactory xxHashFactory) {
	return new ChannelLZ4Decompressor(decompressor, xxHashFactory.newStreamingHash32(DEFAULT_SEED));
}
 

private void receiveMessages() throws IOException
{
    // handshake (true) endpoint versions
    DataOutputStream out = new DataOutputStream(socket.getOutputStream());
    out.writeInt(MessagingService.current_version);
    out.flush();
    DataInputStream in = new DataInputStream(socket.getInputStream());
    int maxVersion = in.readInt();

    from = CompactEndpointSerializationHelper.deserialize(in);
    // record the (true) version of the endpoint
    MessagingService.instance().setVersion(from, maxVersion);
    logger.debug("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from));

    if (compressed)
    {
        logger.debug("Upgrading incoming connection to be compressed");
        if (version < MessagingService.VERSION_21)
        {
            in = new DataInputStream(new SnappyInputStream(socket.getInputStream()));
        }
        else
        {
            LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor();
            Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum();
            in = new DataInputStream(new LZ4BlockInputStream(socket.getInputStream(),
                                                             decompressor,
                                                             checksum));
        }
    }
    else
    {
        in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), BUFFER_SIZE));
    }

    if (version > MessagingService.current_version)
    {
        // save the endpoint so gossip will reconnect to it
        Gossiper.instance.addSavedEndpoint(from);
        logger.info("Received messages from newer protocol version {}. Ignoring", version);
        return;
    }
    // outbound side will reconnect if necessary to upgrade version

    while (true)
    {
        MessagingService.validateMagic(in.readInt());
        receiveMessage(in, version);
    }
}
 

static long hash(byte [] buf, int length) {
    return XXHashFactory.fastestJavaInstance().hash64().hash(buf, 0, length, 0);
}
 

private static String urlToDirName(String uri) {
    return String.valueOf(XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0));
}
 

public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException {
    super(fileReference, filename, type);
    this.file = file;
    channel = Files.newByteChannel(file.toPath());
    this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0);
}
 

public SixtPartitioner() {
    XXHashFactory factory = XXHashFactory.fastestInstance();
    xxHasher = factory.hash32();
}
 

/**
 * Creates a new LZ4 decoder with customizable implementation.创建具有可定制实现的新的LZ4解码器。
 *
 * @param factory            user customizable {@link LZ4Factory} instance
 *                           which may be JNI bindings to the original C implementation, a pure Java implementation
 *                           or a Java implementation that uses the {@link sun.misc.Unsafe}
 * @param validateChecksums  if {@code true}, the checksum field will be validated against the actual
 *                           uncompressed data, and if the checksums do not match, a suitable
 *                           {@link DecompressionException} will be thrown. In this case encoder will use
 *                           xxhash hashing for Java, based on Yann Collet's work available at
 *                           <a href="https://github.com/Cyan4973/xxHash">Github</a>.
 */
public Lz4FrameDecoder(LZ4Factory factory, boolean validateChecksums) {
    this(factory, validateChecksums ?
            XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum()
          : null);
}
 

/**
 * Create a new instance which checks stream integrity using {@link StreamingXXHash32} and doesn't sync flush.
 * 
 * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor, Checksum, boolean)
 * @see StreamingXXHash32#asChecksum()
 */
public Output(OutputStream out, int blockSize, Compressor compressor) {
	this(out, blockSize, DEFAULT_BLOCK_INDEX_SIZE, compressor, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum(), false);
}
 

/**
 * Create a new instance which checks stream integrity using {@link StreamingXXHash32} and doesn't sync flush.
 * 
 * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor, Checksum, boolean)
 * @see StreamingXXHash32#asChecksum()
 */
public Input(InputStream in, Decompressor decompressor) {
	this(in, decompressor, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum());
}
 

/**
 * Creates a new LZ4 encoder with hight or fast compression, default block size (64 KB)
 * and xxhash hashing for Java, based on Yann Collet's work available at
 * <a href="https://github.com/Cyan4973/xxHash">Github</a>.
 *
 * @param highCompressor  if {@code true} codec will use compressor which requires more memory
 *                        and is slower but compresses more efficiently
 *                        基于Yann Collet在Github上的工作，为Java创建一个新的LZ4编码器，它具有较高或快速的压缩、默认块大小(64 KB)和xxhash散列。
 */
public Lz4FrameEncoder(boolean highCompressor) {
    this(LZ4Factory.fastestInstance(), highCompressor, DEFAULT_BLOCK_SIZE,
            XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum());
}