net.jpountz.xxhash.XXHashFactory Java Examples
The following examples show how to use
net.jpountz.xxhash.XXHashFactory.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FileDirectory.java From vespa with Apache License 2.0 | 6 votes |
private Long computeHash(File file) throws IOException { XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); if (file.isDirectory()) { return Files.walk(file.toPath(), 100).map(path -> { try { log.log(Level.FINE, "Calculating hash for '" + path + "'"); return hash(path.toFile(), hasher); } catch (IOException e) { log.log(Level.WARNING, "Failed getting hash from '" + path + "'"); return 0; } }).mapToLong(Number::longValue).sum(); } else { return hash(file, hasher); } }
Example #2
Source File: FileReceiver.java From vespa with Apache License 2.0 | 6 votes |
Session(File downloadDirectory, File tmpDirectory, int sessionId, FileReference reference, FileReferenceData.Type fileType, String fileName, long fileSize) { this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); this.sessionId = sessionId; this.reference = reference; this.fileType = fileType; this.fileName = fileName; this.fileSize = fileSize; currentFileSize = 0; currentPartId = 0; currentHash = 0; fileReferenceDir = new File(downloadDirectory, reference.value()); this.tmpDir = tmpDirectory; try { inprogressFile = Files.createTempFile(tmpDirectory.toPath(), fileName, ".inprogress").toFile(); } catch (IOException e) { String msg = "Failed creating temp file for inprogress file for " + fileName + " in '" + tmpDirectory.toPath() + "': "; log.log(Level.SEVERE, msg + e.getMessage(), e); throw new RuntimeException(msg, e); } }
Example #3
Source File: RpcTester.java From vespa with Apache License 2.0 | 6 votes |
void receive(FileReference reference, String filename, byte[] content) { log.log(Level.INFO, "Preparing receive call for " + reference.value() + " and file " + filename); XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); Request fileBlob = new Request("filedistribution.receiveFile"); log.log(Level.INFO, "Calling " + fileBlob.methodName() + " with target " + target); fileBlob.parameters().add(new StringValue(reference.value())); fileBlob.parameters().add(new StringValue(filename)); fileBlob.parameters().add(new DataValue(content)); fileBlob.parameters().add(new Int64Value(hasher.hash(ByteBuffer.wrap(content), 0))); fileBlob.parameters().add(new Int32Value(0)); fileBlob.parameters().add(new StringValue("OK")); log.log(Level.INFO, "Doing invokeSync"); target.invokeSync(fileBlob, 5); log.log(Level.INFO, "Done with invokeSync"); }
Example #4
Source File: FileDownloaderTest.java From vespa with Apache License 2.0 | 5 votes |
private void receiveFile(FileDownloader fileDownloader, FileReference fileReference, String filename, FileReferenceData.Type type, byte[] content) { XXHash64 hasher = XXHashFactory.fastestInstance().hash64(); FileReceiver.Session session = new FileReceiver.Session(downloadDir, tempDir, 1, fileReference, type, filename, content.length); session.addPart(0, content); File file = session.close(hasher.hash(ByteBuffer.wrap(content), 0)); fileDownloader.fileReferenceDownloader().completedDownloading(fileReference, file); }
Example #5
Source File: UtilsCrypto.java From api with Apache License 2.0 | 5 votes |
/** * Creates a hex number from the input. * From either a `string`, `Uint8Array` or a `Buffer` input, create the xxhash and return the result as a hex number * **example** * <p> * ```java * xxhash64AsValue("abcd", 0xabcd)); // => e29f70f8b8c96df7 * ``` */ //export default function xxhash64AsValue (data: Buffer | Uint8Array | string, seed: number): number { public static long xxhash64AsValue(byte[] data, long seed) { XXHashFactory factory = XXHashFactory.fastestInstance(); ByteArrayInputStream in = new ByteArrayInputStream(data); // value you want, but always the same StreamingXXHash64 xxHash64 = factory.newStreamingHash64(seed); byte[] buf = new byte[16]; // for real-world usage, use a larger buffer, like 8192 bytes for (; ; ) { int read = 0; try { read = in.read(buf); } catch (IOException e) { e.printStackTrace(); } if (read == -1) { break; } xxHash64.update(buf, 0, read); } long hash = xxHash64.getValue(); //System.out.println("value :" + Arrays.toString(data)); //System.out.println("seed :" + seed); //System.out.println("hash :" + Long.toHexString(hash)); return hash; }
Example #6
Source File: Jp32Hasher.java From hash-bench with MIT License | 5 votes |
public static final void register(final Map<String, Hasher> hashers) { hashers.put(Jp32Hasher.XXH32_JNI, new Jp32Hasher(XXHashFactory.nativeInstance().hash32())); hashers.put(Jp32Hasher.XXH32_UNSAFE, new Jp32Hasher(XXHashFactory.unsafeInstance().hash32())); hashers.put(Jp32Hasher.XXH32_SAFE, new Jp32Hasher(XXHashFactory.safeInstance().hash32())); }
Example #7
Source File: SimpleVirtualFilesystem.java From mobi with GNU Affero General Public License v3.0 | 5 votes |
@Activate void activate(Map<String, Object> configuration) throws VirtualFilesystemException { SimpleVirtualFilesystemConfig conf = Configurable.createConfigurable(SimpleVirtualFilesystemConfig.class, configuration); try { this.fsManager = VFS.getManager(); File rootDirectory = new File(conf.defaultRootDirectory()); if (!rootDirectory.exists()) { rootDirectory.mkdirs(); } ((DefaultFileSystemManager) this.fsManager).setBaseFile(rootDirectory); } catch (FileSystemException e) { throw new VirtualFilesystemException("Issue initializing virtual file system.", e); } // Set of queues. tempFiles = new ArrayBlockingQueue<>(conf.maxNumberOfTempFiles()); // Schedule our temp file cleanup service. this.scheduledExecutorService = Executors.newScheduledThreadPool(1); this.scheduledExecutorService.scheduleAtFixedRate(new CleanTempFilesRunnable(LOGGER, tempFiles), conf.secondsBetweenTempCleanup(), conf.secondsBetweenTempCleanup(), TimeUnit.SECONDS); LOGGER.debug("Configured scheduled cleanup of temp files to run every {} seconds", conf.secondsBetweenTempCleanup()); // Set default temp url template this.baseTempUrlTemplate = conf.defaultTemporaryDirectory() != null ? conf.defaultTemporaryDirectory() : ("file://" + System.getProperty("java.io.tmpdir")); LOGGER.debug("Going to use {} for our base temp directory template", this.baseTempUrlTemplate); // Initialize HashFactory this.hashFactory = XXHashFactory.fastestInstance(); }
Example #8
Source File: XXHash32BinaryHashCodeCalculatorTest.java From gridgo with MIT License | 5 votes |
@Test public void calcHashCode() { int id = 10; var hashCodeCalculator = new XXHash32BinaryHashCodeCalculator(XXHashFactory.safeInstance().hash32(), id); var bytes = new byte[] {0x00, 0x01, 0x02, 0x03}; var ans = hashCodeCalculator.calcHashCode(bytes); assertEquals(1256378755, ans); assertEquals(id, hashCodeCalculator.getId()); }
Example #9
Source File: Jp64Hasher.java From hash-bench with MIT License | 5 votes |
public static final void register(final Map<String, Hasher> hashers) { hashers.put(Jp64Hasher.XXH64_JNI, new Jp64Hasher(XXHashFactory.nativeInstance().hash64())); hashers.put(Jp64Hasher.XXH64_UNSAFE, new Jp64Hasher(XXHashFactory.unsafeInstance().hash64())); hashers.put(Jp64Hasher.XXH64_SAFE, new Jp64Hasher(XXHashFactory.safeInstance().hash64())); }
Example #10
Source File: KafkaLZ4BlockInputStream.java From joyqueue with Apache License 2.0 | 5 votes |
/** * Create a new {@link InputStream} that will decompress data using the LZ4 algorithm. * * @param in The stream to decompress * @throws IOException */ public KafkaLZ4BlockInputStream(InputStream in, boolean ignoreFlagDescriptorChecksum) throws IOException { super(in); this.ignoreFlagDescriptorChecksum = ignoreFlagDescriptorChecksum; decompressor = LZ4Factory.fastestInstance().safeDecompressor(); checksum = XXHashFactory.fastestInstance().hash32(); readHeader(); maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[maxBlockSize]; bufferOffset = 0; bufferSize = 0; finished = false; }
Example #11
Source File: KafkaLZ4BlockOutputStream.java From joyqueue with Apache License 2.0 | 5 votes |
/** * Create a new {@link OutputStream} that will compress data using the LZ4 algorithm. * * @param out The output stream to compress * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other values will generate an exception * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for every block of data * @throws IOException */ public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum) throws IOException { super(out); compressor = LZ4Factory.fastestInstance().fastCompressor(); checksum = XXHashFactory.fastestInstance().hash32(); bd = new BD(blockSize); flg = new FLG(blockChecksum); bufferOffset = 0; maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)]; finished = false; writeHeader(); }
Example #12
Source File: KafkaLZ4BlockInputStream.java From joyqueue with Apache License 2.0 | 5 votes |
/** * Create a new {@link java.io.InputStream} that will decompress data using the LZ4 algorithm. * * @param in The stream to decompress * @throws java.io.IOException */ public KafkaLZ4BlockInputStream(InputStream in, boolean ignoreFlagDescriptorChecksum) throws IOException { super(in); this.ignoreFlagDescriptorChecksum = ignoreFlagDescriptorChecksum; decompressor = LZ4Factory.fastestInstance().safeDecompressor(); checksum = XXHashFactory.fastestInstance().hash32(); readHeader(); maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[maxBlockSize]; bufferOffset = 0; bufferSize = 0; finished = false; }
Example #13
Source File: KafkaLZ4BlockOutputStream.java From joyqueue with Apache License 2.0 | 5 votes |
/** * Create a new {@link java.io.OutputStream} that will compress data using the LZ4 algorithm. * * @param out The output stream to compress * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other values will generate an exception * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for every block of data * @throws java.io.IOException */ public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum) throws IOException { super(out); compressor = LZ4Factory.fastestInstance().fastCompressor(); checksum = XXHashFactory.fastestInstance().hash32(); bd = new BD(blockSize); flg = new FLG(blockChecksum); bufferOffset = 0; maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)]; finished = false; writeHeader(); }
Example #14
Source File: KafkaLZ4BlockInputStream.java From joyqueue with Apache License 2.0 | 5 votes |
/** * Create a new {@link InputStream} that will decompress data using the LZ4 algorithm. * * @param in The stream to decompress * @throws IOException */ public KafkaLZ4BlockInputStream(InputStream in, boolean ignoreFlagDescriptorChecksum) throws IOException { super(in); this.ignoreFlagDescriptorChecksum = ignoreFlagDescriptorChecksum; decompressor = LZ4Factory.fastestInstance().safeDecompressor(); checksum = XXHashFactory.fastestInstance().hash32(); readHeader(); maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[maxBlockSize]; bufferOffset = 0; bufferSize = 0; finished = false; }
Example #15
Source File: KafkaLZ4BlockOutputStream.java From joyqueue with Apache License 2.0 | 5 votes |
/** * Create a new {@link OutputStream} that will compress data using the LZ4 algorithm. * * @param out The output stream to compress * @param blockSize Default: 4. The block size used during compression. 4=64kb, 5=256kb, 6=1mb, 7=4mb. All other values will generate an exception * @param blockChecksum Default: false. When true, a XXHash32 checksum is computed and appended to the stream for every block of data * @throws IOException */ public KafkaLZ4BlockOutputStream(OutputStream out, int blockSize, boolean blockChecksum) throws IOException { super(out); compressor = LZ4Factory.fastestInstance().fastCompressor(); checksum = XXHashFactory.fastestInstance().hash32(); bd = new BD(blockSize); flg = new FLG(blockChecksum); bufferOffset = 0; maxBlockSize = bd.getBlockMaximumSize(); buffer = new byte[maxBlockSize]; compressedBuffer = new byte[compressor.maxCompressedLength(maxBlockSize)]; finished = false; writeHeader(); }
Example #16
Source File: Lz4FrameEncoderTest.java From netty-4.1.22 with Apache License 2.0 | 5 votes |
private Lz4FrameEncoder newEncoder(int blockSize, int maxEncodeSize) { Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum(); Lz4FrameEncoder encoder = new Lz4FrameEncoder(LZ4Factory.fastestInstance(), true, blockSize, checksum, maxEncodeSize); encoder.handlerAdded(ctx); return encoder; }
Example #17
Source File: FileDBRegistry.java From vespa with Apache License 2.0 | 5 votes |
private static String uriToRelativeFile(String uri) { String relative = "uri/" + XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0); if (uri.endsWith(".json")) { relative += ".json"; } else if (uri.endsWith(".json.lz4")) { relative += ".json.lz4"; } else if (uri.endsWith(".lz4")) { relative += ".lz4"; } return relative; }
Example #18
Source File: ChannelLZ4Decompressor.java From datakernel with Apache License 2.0 | 4 votes |
public static ChannelLZ4Decompressor create() { return create( LZ4Factory.fastestInstance().fastDecompressor(), XXHashFactory.fastestInstance()); }
Example #19
Source File: FileReferenceDataBlob.java From vespa with Apache License 2.0 | 4 votes |
public FileReferenceDataBlob(FileReference fileReference, String filename, Type type, byte[] content) { this(fileReference, filename, type, content, XXHashFactory.fastestInstance().hash64().hash(ByteBuffer.wrap(content), 0)); }
Example #20
Source File: ChannelLZ4Decompressor.java From datakernel with Apache License 2.0 | 4 votes |
public static ChannelLZ4Decompressor create(LZ4FastDecompressor decompressor, XXHashFactory xxHashFactory) { return new ChannelLZ4Decompressor(decompressor, xxHashFactory.newStreamingHash32(DEFAULT_SEED)); }
Example #21
Source File: IncomingTcpConnection.java From stratio-cassandra with Apache License 2.0 | 4 votes |
private void receiveMessages() throws IOException { // handshake (true) endpoint versions DataOutputStream out = new DataOutputStream(socket.getOutputStream()); out.writeInt(MessagingService.current_version); out.flush(); DataInputStream in = new DataInputStream(socket.getInputStream()); int maxVersion = in.readInt(); from = CompactEndpointSerializationHelper.deserialize(in); // record the (true) version of the endpoint MessagingService.instance().setVersion(from, maxVersion); logger.debug("Set version for {} to {} (will use {})", from, maxVersion, MessagingService.instance().getVersion(from)); if (compressed) { logger.debug("Upgrading incoming connection to be compressed"); if (version < MessagingService.VERSION_21) { in = new DataInputStream(new SnappyInputStream(socket.getInputStream())); } else { LZ4FastDecompressor decompressor = LZ4Factory.fastestInstance().fastDecompressor(); Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(OutboundTcpConnection.LZ4_HASH_SEED).asChecksum(); in = new DataInputStream(new LZ4BlockInputStream(socket.getInputStream(), decompressor, checksum)); } } else { in = new DataInputStream(new BufferedInputStream(socket.getInputStream(), BUFFER_SIZE)); } if (version > MessagingService.current_version) { // save the endpoint so gossip will reconnect to it Gossiper.instance.addSavedEndpoint(from); logger.info("Received messages from newer protocol version {}. Ignoring", version); return; } // outbound side will reconnect if necessary to upgrade version while (true) { MessagingService.validateMagic(in.readInt()); receiveMessage(in, version); } }
Example #22
Source File: SimpleFeeder.java From vespa with Apache License 2.0 | 4 votes |
static long hash(byte [] buf, int length) { return XXHashFactory.fastestJavaInstance().hash64().hash(buf, 0, length, 0); }
Example #23
Source File: UrlDownloadRpcServer.java From vespa with Apache License 2.0 | 4 votes |
private static String urlToDirName(String uri) { return String.valueOf(XXHashFactory.fastestJavaInstance().hash64().hash(ByteBuffer.wrap(Utf8.toBytes(uri)), 0)); }
Example #24
Source File: LazyFileReferenceData.java From vespa with Apache License 2.0 | 4 votes |
public LazyFileReferenceData(FileReference fileReference, String filename, Type type, File file) throws IOException { super(fileReference, filename, type); this.file = file; channel = Files.newByteChannel(file.toPath()); this.hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); }
Example #25
Source File: SixtPartitioner.java From ja-micro with Apache License 2.0 | 4 votes |
public SixtPartitioner() { XXHashFactory factory = XXHashFactory.fastestInstance(); xxHasher = factory.hash32(); }
Example #26
Source File: Lz4FrameDecoder.java From netty-4.1.22 with Apache License 2.0 | 2 votes |
/** * Creates a new LZ4 decoder with customizable implementation.创建具有可定制实现的新的LZ4解码器。 * * @param factory user customizable {@link LZ4Factory} instance * which may be JNI bindings to the original C implementation, a pure Java implementation * or a Java implementation that uses the {@link sun.misc.Unsafe} * @param validateChecksums if {@code true}, the checksum field will be validated against the actual * uncompressed data, and if the checksums do not match, a suitable * {@link DecompressionException} will be thrown. In this case encoder will use * xxhash hashing for Java, based on Yann Collet's work available at * <a href="https://github.com/Cyan4973/xxHash">Github</a>. */ public Lz4FrameDecoder(LZ4Factory factory, boolean validateChecksums) { this(factory, validateChecksums ? XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum() : null); }
Example #27
Source File: CloverDataStream.java From CloverETL-Engine with GNU Lesser General Public License v2.1 | 2 votes |
/** * Create a new instance which checks stream integrity using {@link StreamingXXHash32} and doesn't sync flush. * * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor, Checksum, boolean) * @see StreamingXXHash32#asChecksum() */ public Output(OutputStream out, int blockSize, Compressor compressor) { this(out, blockSize, DEFAULT_BLOCK_INDEX_SIZE, compressor, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum(), false); }
Example #28
Source File: CloverDataStream.java From CloverETL-Engine with GNU Lesser General Public License v2.1 | 2 votes |
/** * Create a new instance which checks stream integrity using {@link StreamingXXHash32} and doesn't sync flush. * * @see #LZ4BlockOutputStream(OutputStream, int, LZ4Compressor, Checksum, boolean) * @see StreamingXXHash32#asChecksum() */ public Input(InputStream in, Decompressor decompressor) { this(in, decompressor, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum()); }
Example #29
Source File: Lz4FrameEncoder.java From netty-4.1.22 with Apache License 2.0 | 2 votes |
/** * Creates a new LZ4 encoder with hight or fast compression, default block size (64 KB) * and xxhash hashing for Java, based on Yann Collet's work available at * <a href="https://github.com/Cyan4973/xxHash">Github</a>. * * @param highCompressor if {@code true} codec will use compressor which requires more memory * and is slower but compresses more efficiently * 基于Yann Collet在Github上的工作,为Java创建一个新的LZ4编码器,它具有较高或快速的压缩、默认块大小(64 KB)和xxhash散列。 */ public Lz4FrameEncoder(boolean highCompressor) { this(LZ4Factory.fastestInstance(), highCompressor, DEFAULT_BLOCK_SIZE, XXHashFactory.fastestInstance().newStreamingHash32(DEFAULT_SEED).asChecksum()); }