com.google.common.hash.Funnel Java Examples

@VisibleForTesting
static <T> CuckooFilter<T> create(Funnel<? super T> funnel, long capacity, double fpp,
                                  CuckooStrategy cuckooStrategy) {
  checkNotNull(funnel);
  checkArgument(capacity > 0, "Expected insertions (%s) must be > 0", capacity);
  checkArgument(fpp > 0.0D, "False positive probability (%s) must be > 0.0", fpp);
  checkArgument(fpp < 1.0D, "False positive probability (%s) must be < 1.0", fpp);
  checkNotNull(cuckooStrategy);

  int numEntriesPerBucket = optimalEntriesPerBucket(fpp);
  long numBuckets = optimalNumberOfBuckets(capacity, numEntriesPerBucket);
  int numBitsPerEntry = optimalBitsPerEntry(fpp, numEntriesPerBucket);

  try {
    return new CuckooFilter<T>(new CuckooTable(numBuckets,
        numEntriesPerBucket, numBitsPerEntry), funnel, cuckooStrategy, fpp);
  } catch (IllegalArgumentException e) {
    throw new IllegalArgumentException("Could not create CuckooFilter of " + numBuckets +
        " buckets, " + numEntriesPerBucket + " entries per bucket, " + numBitsPerEntry +
        " bits per entry", e);
  }
}
 

@VisibleForTesting
static <T> CuckooFilter<T> create(Funnel<? super T> funnel, long capacity, double fpp,
                                  CuckooStrategy cuckooStrategy) {
  checkNotNull(funnel);
  checkArgument(capacity > 0, "Expected insertions (%s) must be > 0", capacity);
  checkArgument(fpp > 0.0D, "False positive probability (%s) must be > 0.0", fpp);
  checkArgument(fpp < 1.0D, "False positive probability (%s) must be < 1.0", fpp);
  checkNotNull(cuckooStrategy);

  int numEntriesPerBucket = optimalEntriesPerBucket(fpp);
  long numBuckets = optimalNumberOfBuckets(capacity, numEntriesPerBucket);
  int numBitsPerEntry = optimalBitsPerEntry(fpp, numEntriesPerBucket);

  try {
    return new CuckooFilter<T>(new CuckooTable(numBuckets,
        numEntriesPerBucket, numBitsPerEntry), funnel, cuckooStrategy, fpp);
  } catch (IllegalArgumentException e) {
    throw new IllegalArgumentException("Could not create CuckooFilter of " + numBuckets +
        " buckets, " + numEntriesPerBucket + " entries per bucket, " + numBitsPerEntry +
        " bits per entry", e);
  }
}
 

@Test
public void ensureGeneric() {
  class SuperClass {
  }
  class SubClass extends SuperClass {
  }

  CuckooFilter<SuperClass> filter = CuckooFilter.create(
      new Funnel<SuperClass>() {
        public void funnel(SuperClass from, PrimitiveSink into) {
          into.putInt(from.hashCode());
        }
      }, 1000, 0.03D);

  assertTrue(filter.add(new SuperClass()));
  assertTrue(filter.add(new SubClass()));
}
 

@Override
public Object getCredentials()
{
    AllowAllUser user = (AllowAllUser) getPrincipal();

    if (user != null) {
        return Hashing.sha256().hashObject(user, new Funnel<AllowAllUser>()
        {
            @Override
            public void funnel(AllowAllUser from, PrimitiveSink into)
            {
                Set<String> fromGroups = from.getGroups();
                String fromName = from.getUserName();

                into.putString(fromName, Charsets.UTF_8);

                for (String fromGroup : fromGroups) {
                    into.putString(fromGroup, Charsets.UTF_8);
                }
            }
        });
    }

    return null;
}
 

private int hash() {
    // Guava documentation recommends using putUnencodedChars to hash raw character bytes within any encoding
    // unless cross-language compatibility is needed. See the Hasher.putString documentation for more info.
    Funnel<TrafficSelector> selectorFunnel = (from, into) -> from.criteria()
            .forEach(c -> into.putUnencodedChars(c.toString()));

    HashFunction hashFunction = Hashing.murmur3_32();
    HashCode hashCode = hashFunction.newHasher()
            .putUnencodedChars(deviceId.toString())
            .putObject(selector, selectorFunnel)
            .putInt(priority)
            .putUnencodedChars(tableId.toString())
            .hash();

    return hashCode.asInt();
}
 

public <T> boolean mightContain(T object, Funnel<? super T> funnel, int numHashFunctions, int[] cells) {
    long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong();
    int hash1 = (int) hash64;
    int hash2 = (int) (hash64 >>> 32);
    for (int i = 1; i <= numHashFunctions; i++) {
        int nextHash = hash1 + i * hash2;
        if (nextHash < 0) {
            nextHash = ~nextHash;
        }
        int pos = nextHash % cells.length;
        if (cells[pos] == 0) {
            return false;
        }
    }
    return true;
}
 

public <T> boolean put(T object, Funnel<? super T> funnel, int numHashFunctions, int[] cells) {
    // TODO(user): when the murmur's shortcuts are implemented, update this code
    long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong();
    int hash1 = (int) hash64;
    int hash2 = (int) (hash64 >>> 32);
    boolean bitsChanged = false;
    for (int i = 1; i <= numHashFunctions; i++) {
        int nextHash = hash1 + i * hash2;
        if (nextHash < 0) {
            nextHash = ~nextHash;
        }
        int pos = nextHash % cells.length;
        bitsChanged |= (cells[pos] != MAX_VAL);
        cells[pos] = MAX_VAL;
    }
    return bitsChanged;
}
 

@Test
public void ensureGeneric() {
  class SuperClass {
  }
  class SubClass extends SuperClass {
  }

  CuckooFilter<SuperClass> filter = CuckooFilter.create(
      new Funnel<SuperClass>() {
        public void funnel(SuperClass from, PrimitiveSink into) {
          into.putInt(from.hashCode());
        }
      }, 1000, 0.03D);

  assertTrue(filter.add(new SuperClass()));
  assertTrue(filter.add(new SubClass()));
}
 

public <T> boolean add(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);

  final long index = index(hash1, table.numBuckets);
  return putEntry(fingerprint, table, index) ||
      putEntry(fingerprint, table, altIndex(index, fingerprint, table.numBuckets));
}
 

@Test
public void customSerialization() throws Exception {
  Funnel<byte[]> funnel = Funnels.byteArrayFunnel();
  CuckooFilter<byte[]> cf = CuckooFilter.create(funnel, 100);
  for (int i = 0; i < 100; i++) {
    cf.add(Ints.toByteArray(i));
  }

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  cf.writeTo(out);

  assertEquals(cf, CuckooFilter.readFrom(new ByteArrayInputStream(out.toByteArray()), funnel));
}
 

public ScalableBloomFilter(final Funnel<? super T> funnel,
                           final int filterCapacity,
                           final double falsePositiveProbability)
{
  checkArgument(filterCapacity > 0, "filter capacity must be greater than 0");
  checkArgument(falsePositiveProbability > 0, "fpp must be greater than 0");

  this.funnel = checkNotNull(funnel);
  this.filterCapacity = filterCapacity;
  this.falsePositiveProbability = falsePositiveProbability;
}
 

private BloomFilter(com.google.common.hash.BloomFilter<E> delegate, Funnel<E> funnel, long capacity, double fpp, long size) {
  super();
  checkNotNull(delegate);
  checkNotNull(funnel);
  checkArgument(capacity >= 0, "capacity must be positive");
  checkArgument(fpp >= 0.0 && fpp < 1.0, "fpp must be positive 0.0 <= fpp < 1.0");
  checkArgument(size >= 0, "size must be positive");
  this.delegate = delegate;
  this.funnel = funnel;
  this.capacity = capacity;
  this.fpp = fpp;
  this.size = size;
}
 

private BloomFilter(com.google.common.hash.BloomFilter<E> delegate, Funnel<E> funnel, long capacity, double fpp, long size) {
  super();
  checkNotNull(delegate);
  checkNotNull(funnel);
  checkArgument(capacity >= 0, "capacity must be positive");
  checkArgument(fpp >= 0.0 && fpp < 1.0, "fpp must be positive 0.0 <= fpp < 1.0");
  checkArgument(size >= 0, "size must be positive");
  this.delegate = delegate;
  this.funnel = funnel;
  this.capacity = capacity;
  this.fpp = fpp;
  this.size = size;
}
 

@Test
public void customSerialization() throws Exception {
  Funnel<byte[]> funnel = Funnels.byteArrayFunnel();
  CuckooFilter<byte[]> cf = CuckooFilter.create(funnel, 100);
  for (int i = 0; i < 100; i++) {
    cf.add(Ints.toByteArray(i));
  }

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  cf.writeTo(out);

  assertEquals(cf, CuckooFilter.readFrom(new ByteArrayInputStream(out.toByteArray()), funnel));
}
 

/**
 * Creates a CuckooFilter.
 */
private CuckooFilter(
    CuckooTable table, Funnel<? super E> funnel, CuckooStrategy cuckooStrategy, double fpp) {
  this.fpp = fpp;
  this.table = checkNotNull(table);
  this.funnel = checkNotNull(funnel);
  this.cuckooStrategy = checkNotNull(cuckooStrategy);
}
 

/**
 * Creates a CuckooFilter.
 */
private CuckooFilter(
    CuckooTable table, Funnel<? super E> funnel, CuckooStrategy cuckooStrategy, double fpp) {
  this.fpp = fpp;
  this.table = checkNotNull(table);
  this.funnel = checkNotNull(funnel);
  this.cuckooStrategy = checkNotNull(cuckooStrategy);
}
 

public <T> boolean contains(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.hasEntry(fingerprint, index1) || table.hasEntry(fingerprint, index2);
}
 

public <T> boolean remove(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index1)
      || table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index2);
}
 

public ConsistentHash(HashFunction hashFunction, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> nodes) {
	this.hashFunction = hashFunction;
	this.nodeFunnel = nodeFunnel;
	this.keyFunnel = keyFunnel;
	for (N node : nodes) {
		add(node);
	}
}
 

/**
 * Creates a new RendezvousHash with a starting set of nodes provided by init. The funnels will be used when generating the hash that combines the nodes and
 * keys. The hasher specifies the hashing algorithm to use.
 */
public RendezvousHash(HashFunction hasher, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> init) {
	if (hasher == null) throw new NullPointerException("hasher");
	if (keyFunnel == null) throw new NullPointerException("keyFunnel");
	if (nodeFunnel == null) throw new NullPointerException("nodeFunnel");
	if (init == null) throw new NullPointerException("init");
	this.hasher = hasher;
	this.keyFunnel = keyFunnel;
	this.nodeFunnel = nodeFunnel;
	this.ordered = new ConcurrentSkipListSet<N>(init);
}
 

public StableBloomFilter(int numCells, int numHashFunctions, int numDecrementCells, Funnel<T> funnel) {
    this.numDecrementCells = numDecrementCells;
    this.cells = new int[numCells];
    this.numHashFunctions = numHashFunctions;
    this.funnel = funnel;
    strategy = new Murmur128_Mitz_32_Strategy();
}
 

static <T> SerializableSaltedHasher<T> create(Algorithm alg, Funnel<? super T> funnel) {
	checkNotNull(alg);
	checkNotNull(funnel);
	SecureRandom randomer = new SecureRandom();
	long seedNSalt = randomer.nextLong();
	long addlSipSeed = randomer.nextLong();
	return new SerializableSaltedHasher<>(seedNSalt, addlSipSeed, funnel, alg);
}
 

SerializableSaltedHasher(long seedNSalt, long addlSipSeed, Funnel<? super T> funnel, Algorithm alg) {
	checkNotNull(alg);
	checkNotNull(funnel);
	this.alg = alg;
	this.funnel = funnel;
	this.seedNSalt = seedNSalt;
	this.addlSipSeed = addlSipSeed;
	hasher = configureHash(alg, seedNSalt, addlSipSeed);
}
 

public <T> boolean contains(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.hasEntry(fingerprint, index1) || table.hasEntry(fingerprint, index2);
}
 

public <T> boolean add(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);

  final long index = index(hash1, table.numBuckets);
  return putEntry(fingerprint, table, index) ||
      putEntry(fingerprint, table, altIndex(index, fingerprint, table.numBuckets));
}
 

/**
 * Create a new instance with the specified history. For instance, from the javadoc above, the constructor would
 * contains {8, 4, 3, 2, 1}.  Shards are returned in the size order they are given in the constructor
 */
public ExpandingShardLocator( final Funnel<T> funnel, final int... bucketSizes ) {

    shardLocatorList = new ShardLocator[bucketSizes.length];

    for ( int i = 0; i < bucketSizes.length; i++ ) {
        shardLocatorList[i] = new ShardLocator<>( funnel, bucketSizes[i] );
    }
}
 

public <T> boolean remove(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index1)
      || table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index2);
}
 

<T> HashCode hash(final T object, final Funnel<? super T> funnel) {
  return hashFunction.hashObject(object, funnel);
}
 

public RendezvousHash(Funnel<N> nodeFunnel, Collection<N> init, int quorum) {
  this.hasher = Hashing.murmur3_128();
  this.nodeFunnel = nodeFunnel;
  this.nodeList = new ConcurrentSkipListSet<>(init);
  this.quorum = quorum;
}
 

/**
 * Reads a byte stream, which was written by {@link #writeTo(OutputStream)}, into a {@link
 * CuckooFilter}. <p/> The {@code Funnel} to be used is not encoded in the stream, so it must be
 * provided here. <b>Warning:</b> the funnel provided <b>must</b> behave identically to the one
 * used to populate the original Cuckoo filter!
 *
 * @throws IOException if the InputStream throws an {@code IOException}, or if its data does not
 *                     appear to be a CuckooFilter serialized using the {@link
 *                     #writeTo(OutputStream)} method.
 */
@CheckReturnValue
public static <T> CuckooFilter<T> readFrom(InputStream in, Funnel<T> funnel) throws IOException {
  checkNotNull(in, "InputStream");
  checkNotNull(funnel, "Funnel");
  int strategyOrdinal = -1;
  double fpp = -1.0D;
  long size = -1L;
  long checksum = -1L;
  long numBuckets = -1L;
  int numEntriesPerBucket = -1;
  int numBitsPerEntry = -1;
  int dataLength = -1;
  try {
    DataInputStream din = new DataInputStream(in);
    // currently this assumes there is no negative ordinal; will have to be updated if we
    // add non-stateless strategies (for which we've reserved negative ordinals; see
    // Strategy.ordinal()).
    strategyOrdinal = din.readByte();
    fpp = din.readDouble();
    size = din.readLong();
    checksum = din.readLong();
    numBuckets = din.readLong();
    numEntriesPerBucket = din.readInt();
    numBitsPerEntry = din.readInt();
    dataLength = din.readInt();

    CuckooStrategy cuckooStrategy = CuckooStrategies.values()[strategyOrdinal].strategy();
    long[] data = new long[dataLength];
    for (int i = 0; i < data.length; i++) {
      data[i] = din.readLong();
    }
    return new CuckooFilter<T>(
        new CuckooTable(data, size, checksum, numBuckets, numEntriesPerBucket, numBitsPerEntry),
        funnel, cuckooStrategy, fpp);
  } catch (RuntimeException e) {
    IOException ioException = new IOException(
        "Unable to deserialize CuckooFilter from InputStream."
            + " strategyOrdinal: " + strategyOrdinal
            + " fpp: " + fpp
            + " size: " + size
            + " checksum: " + checksum
            + " numBuckets: " + numBuckets
            + " numEntriesPerBucket: " + numEntriesPerBucket
            + " numBitsPerEntry: " + numBitsPerEntry
            + " dataLength: " + dataLength);
    ioException.initCause(e);
    throw ioException;
  }
}