com.google.common.hash.Funnel Java Examples

The following examples show how to use com.google.common.hash.Funnel. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CuckooFilter.java    From guava-probably with Apache License 2.0 7 votes vote down vote up
@VisibleForTesting
static <T> CuckooFilter<T> create(Funnel<? super T> funnel, long capacity, double fpp,
                                  CuckooStrategy cuckooStrategy) {
  checkNotNull(funnel);
  checkArgument(capacity > 0, "Expected insertions (%s) must be > 0", capacity);
  checkArgument(fpp > 0.0D, "False positive probability (%s) must be > 0.0", fpp);
  checkArgument(fpp < 1.0D, "False positive probability (%s) must be < 1.0", fpp);
  checkNotNull(cuckooStrategy);

  int numEntriesPerBucket = optimalEntriesPerBucket(fpp);
  long numBuckets = optimalNumberOfBuckets(capacity, numEntriesPerBucket);
  int numBitsPerEntry = optimalBitsPerEntry(fpp, numEntriesPerBucket);

  try {
    return new CuckooFilter<T>(new CuckooTable(numBuckets,
        numEntriesPerBucket, numBitsPerEntry), funnel, cuckooStrategy, fpp);
  } catch (IllegalArgumentException e) {
    throw new IllegalArgumentException("Could not create CuckooFilter of " + numBuckets +
        " buckets, " + numEntriesPerBucket + " entries per bucket, " + numBitsPerEntry +
        " bits per entry", e);
  }
}
 
Example #2
Source File: CuckooFilter.java    From guava-probably with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
static <T> CuckooFilter<T> create(Funnel<? super T> funnel, long capacity, double fpp,
                                  CuckooStrategy cuckooStrategy) {
  checkNotNull(funnel);
  checkArgument(capacity > 0, "Expected insertions (%s) must be > 0", capacity);
  checkArgument(fpp > 0.0D, "False positive probability (%s) must be > 0.0", fpp);
  checkArgument(fpp < 1.0D, "False positive probability (%s) must be < 1.0", fpp);
  checkNotNull(cuckooStrategy);

  int numEntriesPerBucket = optimalEntriesPerBucket(fpp);
  long numBuckets = optimalNumberOfBuckets(capacity, numEntriesPerBucket);
  int numBitsPerEntry = optimalBitsPerEntry(fpp, numEntriesPerBucket);

  try {
    return new CuckooFilter<T>(new CuckooTable(numBuckets,
        numEntriesPerBucket, numBitsPerEntry), funnel, cuckooStrategy, fpp);
  } catch (IllegalArgumentException e) {
    throw new IllegalArgumentException("Could not create CuckooFilter of " + numBuckets +
        " buckets, " + numEntriesPerBucket + " entries per bucket, " + numBitsPerEntry +
        " bits per entry", e);
  }
}
 
Example #3
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 6 votes vote down vote up
@Test
public void ensureGeneric() {
  class SuperClass {
  }
  class SubClass extends SuperClass {
  }

  CuckooFilter<SuperClass> filter = CuckooFilter.create(
      new Funnel<SuperClass>() {
        public void funnel(SuperClass from, PrimitiveSink into) {
          into.putInt(from.hashCode());
        }
      }, 1000, 0.03D);

  assertTrue(filter.add(new SuperClass()));
  assertTrue(filter.add(new SubClass()));
}
 
Example #4
Source File: AllowAllToken.java    From airpal with Apache License 2.0 6 votes vote down vote up
@Override
public Object getCredentials()
{
    AllowAllUser user = (AllowAllUser) getPrincipal();

    if (user != null) {
        return Hashing.sha256().hashObject(user, new Funnel<AllowAllUser>()
        {
            @Override
            public void funnel(AllowAllUser from, PrimitiveSink into)
            {
                Set<String> fromGroups = from.getGroups();
                String fromName = from.getUserName();

                into.putString(fromName, Charsets.UTF_8);

                for (String fromGroup : fromGroups) {
                    into.putString(fromGroup, Charsets.UTF_8);
                }
            }
        });
    }

    return null;
}
 
Example #5
Source File: DefaultFlowRule.java    From onos with Apache License 2.0 6 votes vote down vote up
private int hash() {
    // Guava documentation recommends using putUnencodedChars to hash raw character bytes within any encoding
    // unless cross-language compatibility is needed. See the Hasher.putString documentation for more info.
    Funnel<TrafficSelector> selectorFunnel = (from, into) -> from.criteria()
            .forEach(c -> into.putUnencodedChars(c.toString()));

    HashFunction hashFunction = Hashing.murmur3_32();
    HashCode hashCode = hashFunction.newHasher()
            .putUnencodedChars(deviceId.toString())
            .putObject(selector, selectorFunnel)
            .putInt(priority)
            .putUnencodedChars(tableId.toString())
            .hash();

    return hashCode.asInt();
}
 
Example #6
Source File: StableBloomFilter.java    From streaminer with Apache License 2.0 6 votes vote down vote up
public <T> boolean mightContain(T object, Funnel<? super T> funnel, int numHashFunctions, int[] cells) {
    long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong();
    int hash1 = (int) hash64;
    int hash2 = (int) (hash64 >>> 32);
    for (int i = 1; i <= numHashFunctions; i++) {
        int nextHash = hash1 + i * hash2;
        if (nextHash < 0) {
            nextHash = ~nextHash;
        }
        int pos = nextHash % cells.length;
        if (cells[pos] == 0) {
            return false;
        }
    }
    return true;
}
 
Example #7
Source File: StableBloomFilter.java    From streaminer with Apache License 2.0 6 votes vote down vote up
public <T> boolean put(T object, Funnel<? super T> funnel, int numHashFunctions, int[] cells) {
    // TODO(user): when the murmur's shortcuts are implemented, update this code
    long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong();
    int hash1 = (int) hash64;
    int hash2 = (int) (hash64 >>> 32);
    boolean bitsChanged = false;
    for (int i = 1; i <= numHashFunctions; i++) {
        int nextHash = hash1 + i * hash2;
        if (nextHash < 0) {
            nextHash = ~nextHash;
        }
        int pos = nextHash % cells.length;
        bitsChanged |= (cells[pos] != MAX_VAL);
        cells[pos] = MAX_VAL;
    }
    return bitsChanged;
}
 
Example #8
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 6 votes vote down vote up
@Test
public void ensureGeneric() {
  class SuperClass {
  }
  class SubClass extends SuperClass {
  }

  CuckooFilter<SuperClass> filter = CuckooFilter.create(
      new Funnel<SuperClass>() {
        public void funnel(SuperClass from, PrimitiveSink into) {
          into.putInt(from.hashCode());
        }
      }, 1000, 0.03D);

  assertTrue(filter.add(new SuperClass()));
  assertTrue(filter.add(new SubClass()));
}
 
Example #9
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
public <T> boolean add(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);

  final long index = index(hash1, table.numBuckets);
  return putEntry(fingerprint, table, index) ||
      putEntry(fingerprint, table, altIndex(index, fingerprint, table.numBuckets));
}
 
Example #10
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void customSerialization() throws Exception {
  Funnel<byte[]> funnel = Funnels.byteArrayFunnel();
  CuckooFilter<byte[]> cf = CuckooFilter.create(funnel, 100);
  for (int i = 0; i < 100; i++) {
    cf.add(Ints.toByteArray(i));
  }

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  cf.writeTo(out);

  assertEquals(cf, CuckooFilter.readFrom(new ByteArrayInputStream(out.toByteArray()), funnel));
}
 
Example #11
Source File: ScalableBloomFilter.java    From nexus-public with Eclipse Public License 1.0 5 votes vote down vote up
public ScalableBloomFilter(final Funnel<? super T> funnel,
                           final int filterCapacity,
                           final double falsePositiveProbability)
{
  checkArgument(filterCapacity > 0, "filter capacity must be greater than 0");
  checkArgument(falsePositiveProbability > 0, "fpp must be greater than 0");

  this.funnel = checkNotNull(funnel);
  this.filterCapacity = filterCapacity;
  this.falsePositiveProbability = falsePositiveProbability;
}
 
Example #12
Source File: BloomFilter.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
private BloomFilter(com.google.common.hash.BloomFilter<E> delegate, Funnel<E> funnel, long capacity, double fpp, long size) {
  super();
  checkNotNull(delegate);
  checkNotNull(funnel);
  checkArgument(capacity >= 0, "capacity must be positive");
  checkArgument(fpp >= 0.0 && fpp < 1.0, "fpp must be positive 0.0 <= fpp < 1.0");
  checkArgument(size >= 0, "size must be positive");
  this.delegate = delegate;
  this.funnel = funnel;
  this.capacity = capacity;
  this.fpp = fpp;
  this.size = size;
}
 
Example #13
Source File: BloomFilter.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
private BloomFilter(com.google.common.hash.BloomFilter<E> delegate, Funnel<E> funnel, long capacity, double fpp, long size) {
  super();
  checkNotNull(delegate);
  checkNotNull(funnel);
  checkArgument(capacity >= 0, "capacity must be positive");
  checkArgument(fpp >= 0.0 && fpp < 1.0, "fpp must be positive 0.0 <= fpp < 1.0");
  checkArgument(size >= 0, "size must be positive");
  this.delegate = delegate;
  this.funnel = funnel;
  this.capacity = capacity;
  this.fpp = fpp;
  this.size = size;
}
 
Example #14
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void customSerialization() throws Exception {
  Funnel<byte[]> funnel = Funnels.byteArrayFunnel();
  CuckooFilter<byte[]> cf = CuckooFilter.create(funnel, 100);
  for (int i = 0; i < 100; i++) {
    cf.add(Ints.toByteArray(i));
  }

  ByteArrayOutputStream out = new ByteArrayOutputStream();
  cf.writeTo(out);

  assertEquals(cf, CuckooFilter.readFrom(new ByteArrayInputStream(out.toByteArray()), funnel));
}
 
Example #15
Source File: CuckooFilter.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a CuckooFilter.
 */
private CuckooFilter(
    CuckooTable table, Funnel<? super E> funnel, CuckooStrategy cuckooStrategy, double fpp) {
  this.fpp = fpp;
  this.table = checkNotNull(table);
  this.funnel = checkNotNull(funnel);
  this.cuckooStrategy = checkNotNull(cuckooStrategy);
}
 
Example #16
Source File: CuckooFilter.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a CuckooFilter.
 */
private CuckooFilter(
    CuckooTable table, Funnel<? super E> funnel, CuckooStrategy cuckooStrategy, double fpp) {
  this.fpp = fpp;
  this.table = checkNotNull(table);
  this.funnel = checkNotNull(funnel);
  this.cuckooStrategy = checkNotNull(cuckooStrategy);
}
 
Example #17
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
public <T> boolean contains(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.hasEntry(fingerprint, index1) || table.hasEntry(fingerprint, index2);
}
 
Example #18
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
public <T> boolean remove(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index1)
      || table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index2);
}
 
Example #19
Source File: ConsistentHash.java    From RendezvousHash with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public ConsistentHash(HashFunction hashFunction, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> nodes) {
	this.hashFunction = hashFunction;
	this.nodeFunnel = nodeFunnel;
	this.keyFunnel = keyFunnel;
	for (N node : nodes) {
		add(node);
	}
}
 
Example #20
Source File: RendezvousHash.java    From RendezvousHash with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Creates a new RendezvousHash with a starting set of nodes provided by init. The funnels will be used when generating the hash that combines the nodes and
 * keys. The hasher specifies the hashing algorithm to use.
 */
public RendezvousHash(HashFunction hasher, Funnel<K> keyFunnel, Funnel<N> nodeFunnel, Collection<N> init) {
	if (hasher == null) throw new NullPointerException("hasher");
	if (keyFunnel == null) throw new NullPointerException("keyFunnel");
	if (nodeFunnel == null) throw new NullPointerException("nodeFunnel");
	if (init == null) throw new NullPointerException("init");
	this.hasher = hasher;
	this.keyFunnel = keyFunnel;
	this.nodeFunnel = nodeFunnel;
	this.ordered = new ConcurrentSkipListSet<N>(init);
}
 
Example #21
Source File: StableBloomFilter.java    From streaminer with Apache License 2.0 5 votes vote down vote up
public StableBloomFilter(int numCells, int numHashFunctions, int numDecrementCells, Funnel<T> funnel) {
    this.numDecrementCells = numDecrementCells;
    this.cells = new int[numCells];
    this.numHashFunctions = numHashFunctions;
    this.funnel = funnel;
    strategy = new Murmur128_Mitz_32_Strategy();
}
 
Example #22
Source File: SerializableSaltedHasher.java    From CuckooFilter4J with Apache License 2.0 5 votes vote down vote up
static <T> SerializableSaltedHasher<T> create(Algorithm alg, Funnel<? super T> funnel) {
	checkNotNull(alg);
	checkNotNull(funnel);
	SecureRandom randomer = new SecureRandom();
	long seedNSalt = randomer.nextLong();
	long addlSipSeed = randomer.nextLong();
	return new SerializableSaltedHasher<>(seedNSalt, addlSipSeed, funnel, alg);
}
 
Example #23
Source File: SerializableSaltedHasher.java    From CuckooFilter4J with Apache License 2.0 5 votes vote down vote up
SerializableSaltedHasher(long seedNSalt, long addlSipSeed, Funnel<? super T> funnel, Algorithm alg) {
	checkNotNull(alg);
	checkNotNull(funnel);
	this.alg = alg;
	this.funnel = funnel;
	this.seedNSalt = seedNSalt;
	this.addlSipSeed = addlSipSeed;
	hasher = configureHash(alg, seedNSalt, addlSipSeed);
}
 
Example #24
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
public <T> boolean contains(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.hasEntry(fingerprint, index1) || table.hasEntry(fingerprint, index2);
}
 
Example #25
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
public <T> boolean add(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);

  final long index = index(hash1, table.numBuckets);
  return putEntry(fingerprint, table, index) ||
      putEntry(fingerprint, table, altIndex(index, fingerprint, table.numBuckets));
}
 
Example #26
Source File: ExpandingShardLocator.java    From usergrid with Apache License 2.0 5 votes vote down vote up
/**
 * Create a new instance with the specified history. For instance, from the javadoc above, the constructor would
 * contains {8, 4, 3, 2, 1}.  Shards are returned in the size order they are given in the constructor
 */
public ExpandingShardLocator( final Funnel<T> funnel, final int... bucketSizes ) {

    shardLocatorList = new ShardLocator[bucketSizes.length];

    for ( int i = 0; i < bucketSizes.length; i++ ) {
        shardLocatorList[i] = new ShardLocator<>( funnel, bucketSizes[i] );
    }
}
 
Example #27
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
public <T> boolean remove(T object, Funnel<? super T> funnel, CuckooTable table) {
  final long hash64 = hash(object, funnel).asLong();
  final int hash1 = hash1(hash64);
  final int hash2 = hash2(hash64);
  final int fingerprint = fingerprint(hash2, table.numBitsPerEntry);
  final long index1 = index(hash1, table.numBuckets);
  final long index2 = altIndex(index1, fingerprint, table.numBuckets);
  return table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index1)
      || table.swapAnyEntry(CuckooTable.EMPTY_ENTRY, fingerprint, index2);
}
 
Example #28
Source File: CuckooStrategyMurmurBealDupras32.java    From guava-probably with Apache License 2.0 4 votes vote down vote up
<T> HashCode hash(final T object, final Funnel<? super T> funnel) {
  return hashFunction.hashObject(object, funnel);
}
 
Example #29
Source File: RendezvousHash.java    From xio with Apache License 2.0 4 votes vote down vote up
public RendezvousHash(Funnel<N> nodeFunnel, Collection<N> init, int quorum) {
  this.hasher = Hashing.murmur3_128();
  this.nodeFunnel = nodeFunnel;
  this.nodeList = new ConcurrentSkipListSet<>(init);
  this.quorum = quorum;
}
 
Example #30
Source File: CuckooFilter.java    From guava-probably with Apache License 2.0 4 votes vote down vote up
/**
 * Reads a byte stream, which was written by {@link #writeTo(OutputStream)}, into a {@link
 * CuckooFilter}. <p/> The {@code Funnel} to be used is not encoded in the stream, so it must be
 * provided here. <b>Warning:</b> the funnel provided <b>must</b> behave identically to the one
 * used to populate the original Cuckoo filter!
 *
 * @throws IOException if the InputStream throws an {@code IOException}, or if its data does not
 *                     appear to be a CuckooFilter serialized using the {@link
 *                     #writeTo(OutputStream)} method.
 */
@CheckReturnValue
public static <T> CuckooFilter<T> readFrom(InputStream in, Funnel<T> funnel) throws IOException {
  checkNotNull(in, "InputStream");
  checkNotNull(funnel, "Funnel");
  int strategyOrdinal = -1;
  double fpp = -1.0D;
  long size = -1L;
  long checksum = -1L;
  long numBuckets = -1L;
  int numEntriesPerBucket = -1;
  int numBitsPerEntry = -1;
  int dataLength = -1;
  try {
    DataInputStream din = new DataInputStream(in);
    // currently this assumes there is no negative ordinal; will have to be updated if we
    // add non-stateless strategies (for which we've reserved negative ordinals; see
    // Strategy.ordinal()).
    strategyOrdinal = din.readByte();
    fpp = din.readDouble();
    size = din.readLong();
    checksum = din.readLong();
    numBuckets = din.readLong();
    numEntriesPerBucket = din.readInt();
    numBitsPerEntry = din.readInt();
    dataLength = din.readInt();

    CuckooStrategy cuckooStrategy = CuckooStrategies.values()[strategyOrdinal].strategy();
    long[] data = new long[dataLength];
    for (int i = 0; i < data.length; i++) {
      data[i] = din.readLong();
    }
    return new CuckooFilter<T>(
        new CuckooTable(data, size, checksum, numBuckets, numEntriesPerBucket, numBitsPerEntry),
        funnel, cuckooStrategy, fpp);
  } catch (RuntimeException e) {
    IOException ioException = new IOException(
        "Unable to deserialize CuckooFilter from InputStream."
            + " strategyOrdinal: " + strategyOrdinal
            + " fpp: " + fpp
            + " size: " + size
            + " checksum: " + checksum
            + " numBuckets: " + numBuckets
            + " numEntriesPerBucket: " + numEntriesPerBucket
            + " numBitsPerEntry: " + numBitsPerEntry
            + " dataLength: " + dataLength);
    ioException.initCause(e);
    throw ioException;
  }
}