com.google.common.hash.Funnels Java Examples

The following examples show how to use com.google.common.hash.Funnels. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void testEquals() {
	CuckooFilter<Integer> partFull = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	CuckooFilter<Integer> full = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	for (int i = 0; i < 1000000; i++) {
		assertTrue(partFull.put(i));
	}
	for (int i = 0;; i++) {
		if (!full.put(i))
			break;
	}
	new EqualsTester().addEqualityGroup(partFull).addEqualityGroup(full)
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
					.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.longFunnel(), 2000000).withFalsePositiveRate(0.01)
					.withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 1000000)
					.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
					.withFalsePositiveRate(0.03).withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
					.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_128).build())
			.testEquals();
}
 
Example #2
Source File: HashedFileContent.java    From n4js with Eclipse Public License 1.0 6 votes vote down vote up
/** Create a fingerprint of the given file at the given location. */
public HashedFileContent(URI uri, File file) throws IOException {
	this.uri = uri;
	String ext = uri.fileExtension();
	if (ext == null || "ts".equals(ext) || "js".equals(ext) || "jsx".equals(ext) || "map".equals(ext)
			|| "md".equals(ext)
			|| "hbs".equals(ext)
			|| "json".equals(ext) && !"package.json".equals(uri.lastSegment())) {
		this.hash = file.length();
	} else {
		try (InputStream s = new FileInputStream(file)) {
			Hasher hasher = hashFunction.newHasher();
			s.transferTo(Funnels.asOutputStream(hasher));
			this.hash = hasher.hash().asLong();
		}
	}
}
 
Example #3
Source File: BlacklistPasswordPolicyProviderFactory.java    From keycloak with Apache License 2.0 6 votes vote down vote up
/**
 * Loads the referenced blacklist into a {@link BloomFilter}.
 *
 * @return the {@link BloomFilter} backing a password blacklist
 */
private BloomFilter<String> load() {

    try {
        LOG.infof("Loading blacklist with name %s from %s - start", name, path);

        long passwordCount = getPasswordCount();

        BloomFilter<String> filter = BloomFilter.create(
                Funnels.stringFunnel(StandardCharsets.UTF_8),
                passwordCount,
                FALSE_POSITIVE_PROBABILITY);

        try (BufferedReader br = newReader(path)) {
            br.lines().forEach(filter::put);
        }

        LOG.infof("Loading blacklist with name %s from %s - end", name, path);

        return filter;
    } catch (IOException e) {
        throw new RuntimeException("Could not load password blacklist from path: " + path, e);
    }
}
 
Example #4
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 6 votes vote down vote up
@Test
public void addAll() {
  int element1 = 1;
  int element2 = 2;

  CuckooFilter<Integer> cf1 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf1.add(element1);
  assertTrue(cf1.contains(element1));
  assertFalse(cf1.contains(element2));

  CuckooFilter<Integer> cf2 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf2.add(element2);
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));

  assertTrue(cf1.isCompatible(cf2));
  cf1.addAll(cf2);
  assertTrue(cf1.contains(element1));
  assertTrue(cf1.contains(element2));
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));
}
 
Example #5
Source File: XpTrackerService.java    From runelite with BSD 2-Clause "Simplified" License 6 votes vote down vote up
private BloomFilter<String> createFilter()
{
	final BloomFilter<String> filter = BloomFilter.create(
		Funnels.stringFunnel(Charset.defaultCharset()),
		BLOOMFILTER_EXPECTED_INSERTIONS
	);

	synchronized (usernameUpdateQueue)
	{
		for (String toUpdate : usernameUpdateQueue)
		{
			filter.put(toUpdate);
		}
	}

	return filter;
}
 
Example #6
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 6 votes vote down vote up
@Test
public void addAll() {
  int element1 = 1;
  int element2 = 2;

  CuckooFilter<Integer> cf1 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf1.add(element1);
  assertTrue(cf1.contains(element1));
  assertFalse(cf1.contains(element2));

  CuckooFilter<Integer> cf2 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf2.add(element2);
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));

  assertTrue(cf1.isCompatible(cf2));
  cf1.addAll(cf2);
  assertTrue(cf1.contains(element1));
  assertTrue(cf1.contains(element2));
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));
}
 
Example #7
Source File: TestIndexTagCalc.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void knownZeroTags32() {
	// manual instantiation to force salts to be static
	SerializableSaltedHasher<Integer> hasher = new SerializableSaltedHasher<>(0, 0, Funnels.integerFunnel(),
			Algorithm.Murmur3_32);
	IndexTagCalc<Integer> indexer = new IndexTagCalc<>(hasher, 128, 4);
	// find some 0 value tags
	int zeroTags = 0;
	int i = 0;
	ArrayList<Integer> zeroTagInputs = new ArrayList<>();
	while (zeroTags < 20) {
		if (indexer.getTagValue32(hasher.hashObj(i).asInt()) == 0) {
			zeroTagInputs.add(i);
			zeroTags++;
		}
		i++;
	}
	for (Integer tag : zeroTagInputs) {
		// none of the zero value tags should be zero from indexer if
		// rehashing is working properly
		assertFalse(indexer.generate(tag).tag == 0);
	}

}
 
Example #8
Source File: TestIndexTagCalc.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void sanityTagIndexBitsUsed64() {
	// manual instantiation to force salts to be static
	SerializableSaltedHasher<Integer> hasher = new SerializableSaltedHasher<>(0, 0, Funnels.integerFunnel(),
			Algorithm.sipHash24);
	IndexTagCalc<Integer> indexer = new IndexTagCalc<>(hasher, (long) Math.pow(2, 31), 32);
	long setBitsIndex = 0;
	long setBitsTag = 0;
	// should be enough to set all bits being used...
	for (int i = 0; i < 1234567; i++) {
		BucketAndTag bt = indexer.generate(i);
		setBitsIndex |= bt.index;
		setBitsTag |= bt.tag;
	}
	// will be true if we're using the right number of bits for tag and
	// index for this calculator
	assertTrue(Long.bitCount(setBitsIndex) == 31);
	assertTrue(Long.bitCount(setBitsTag) == 32);
	// check where the set bits are
	long bitMask32 = -1L >>> 32;// (mask for lower 32 bits set)
	long bitMask31 = bitMask32 >>> 1;// (mask for lower 32 bits set)
	assertTrue(bitMask32 == setBitsTag);
	assertTrue(bitMask31 == setBitsIndex);
}
 
Example #9
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFalseNegative() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	// add them to filter
	for (int i = 0; i < 100000; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false negatives
	int falseNegatives = 0;
	for (int i = 0; i < 100000; i++) {
		if (!filter.mightContain(i)) {
			falseNegatives++;
		}
	}
	assertTrue(falseNegatives + " false negatives detected", falseNegatives == 0);

}
 
Example #10
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFailedDelete() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();

	// make a list of test values(all unique)
	int maxInsertedVal = 100000;
	for (int i = 0; i < maxInsertedVal; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false deletes(if I can't delete something that's definitely
	// there)
	int falseDeletes = 0;
	for (int i = 0; i < maxInsertedVal; i++) {
		if (!filter.delete(i)) {
			falseDeletes++;
		}
	}
	assertTrue(falseDeletes + " false deletions detected", falseDeletes == 0);
}
 
Example #11
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFalseDeleteRate() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	int maxInsertedVal = 100000;
	for (int i = 0; i < maxInsertedVal; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false delete rate(deleted something I didn't add
	// successfully)
	int falseDeletes = 0;
	// false delete rate should roughly match false positive rate
	int totalAttempts = 10000;
	maxInsertedVal += 1;
	for (int i = maxInsertedVal; i < totalAttempts + maxInsertedVal; i++) {
		if (filter.delete(i))
			falseDeletes++;
	}
	assertTrue(
			falseDeletes
					+ " false deletions detected. False delete rate is above 0.02 on filter with 0.01 false positive rate",
			(double) falseDeletes / totalAttempts < 0.02);

}
 
Example #12
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFalsePositiveRate() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	int maxInsertedVal = 100000;
	// make a list of test values(all unique)
	for (int i = 0; i < maxInsertedVal; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false positive rate(contains something I didn't add)
	int falsePositives = 0;
	maxInsertedVal += 1;
	int totalAttempts = 100000;
	for (int i = maxInsertedVal; i < totalAttempts + maxInsertedVal; i++) {
		if (filter.mightContain(i))
			falsePositives++;
	}
	assertTrue((double) falsePositives / totalAttempts + " false positive rate is above limit",
			(double) falsePositives / totalAttempts < 0.02);

}
 
Example #13
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 6 votes vote down vote up
@Test
public void sanityTestVictimCache() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();

	for (int i = 0; i < 9; i++) {
		assertTrue(filter.put(42));
	}
	assertTrue(filter.getCount() == 9);
	for (int i = 0; i < 9; i++) {
		assertTrue(filter.mightContain(42));
		assertTrue(filter.delete(42));
	}
	assertFalse(filter.delete(42));
	assertFalse(filter.mightContain(42));
	assertTrue(filter.getCount() == 0);
	// at this point victim cache is in use since both buckets for 42 are
	// full

}
 
Example #14
Source File: BloomFilterUnitTest.java    From tutorials with MIT License 6 votes vote down vote up
@Test
public void givenBloomFilter_whenAddNStringsToIt_thenShouldNotReturnAnyFalsePositive() {
    //when
    BloomFilter<Integer> filter = BloomFilter.create(
            Funnels.integerFunnel(),
            500,
            0.01);

    //when
    filter.put(1);
    filter.put(2);
    filter.put(3);

    //then
    // the probability that it returns true, but is actually false is 1%
    assertThat(filter.mightContain(1)).isTrue();
    assertThat(filter.mightContain(2)).isTrue();
    assertThat(filter.mightContain(3)).isTrue();

    assertThat(filter.mightContain(100)).isFalse();
}
 
Example #15
Source File: CompromisedPasswordClassifierTest.java    From quarantyne with Apache License 2.0 5 votes vote down vote up
@Test
public void testClassifier() {
  BloomFilter<String> bloom =
      BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 3);
  bloom.put("alpha");
  bloom.put("bravo");
  bloom.put("charlie");

  Supplier<Config> configSupplier = () -> Config.builder()
      .loginAction(new QIdentityAction("/login", "email", "password"))
      .registerAction(new QIdentityAction("/register", "email", "password"))
      .build();
  CompromisedPasswordClassifier classifier = new CompromisedPasswordClassifier(bloom, configSupplier);
  HttpRequest defaultRequest = TestHttpRequest.REQ();

  // null empty
  assertThat(classifier.classify(defaultRequest, null)).isEqualTo(Label.NONE);
  assertThat(classifier.classify(defaultRequest, TestHttpRequestBody.EMPTY)).isEqualTo(Label.NONE);

  // no key matches password
  assertThat(classifier.classify(defaultRequest,
      TestHttpRequestBody.make(new JsonObject().put("name", "john")))).isEqualTo(Label.NONE);

  // a key matches password but password is not in bloomf
  assertThat(classifier.classify(defaultRequest,
      TestHttpRequestBody.make(new JsonObject().put("password", "delta")))).isEqualTo(Label.NONE);

  // match
  HttpRequest requestOnPath = new TestHttpRequest.Builder().setPath("/login").build();
  assertThat(classifier.classify(requestOnPath,
      TestHttpRequestBody.make(new JsonObject().put("password", "bravo")))).isEqualTo(
          Label.COMPROMISED_PASSWORD);
}
 
Example #16
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 5 votes vote down vote up
@Test
public void testVictimCacheTagComparison() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	filter.hasVictim = true;
	filter.victim = new Victim(1, 2, 42);
	BucketAndTag test1 = new BucketAndTag(filter.victim.getI1(), 42);
	BucketAndTag test2 = new BucketAndTag(filter.victim.getI2(), 42);
	assertTrue(filter.checkIsVictim(test1));
	assertTrue(filter.checkIsVictim(test2));
}
 
Example #17
Source File: ByteSource.java    From codebuff with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * Hashes the contents of this byte source using the given hash function.
 *
 * @throws IOException if an I/O error occurs in the process of reading from this source
 */


public HashCode hash(HashFunction hashFunction) throws IOException {
  Hasher hasher = hashFunction.newHasher();
  copyTo(Funnels.asOutputStream(hasher));
  return hasher.hash();
}
 
Example #18
Source File: Bloom.java    From bidder with Apache License 2.0 5 votes vote down vote up
/**
 * Reads a file or S3 object line by line and loads the filter.
 * @param br BufferedReader. The line-by-line reader.
 * @throws Exception on I/O errors.
 */
void makeFilter(BufferedReader br, long size) throws Exception {
	String[] parts;
	int i;
	long sz;
	
	double fpp = 0.003; // desired false positive probability
	
	String line = br.readLine();
	sz = line.length() - 5;
	if (sz < 0)
		sz = 16;                // just a guess
	sz = size / sz;
	sz *= 2;
	parts = eatquotedStrings(line);
	this.size = 1;
	for (i = 0; i < parts.length; i++) {
		parts[i] = parts[i].replaceAll("\"", "");
	}
	
	bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charset.forName("UTF-8")), sz,fpp);
	bloomFilter.put(parts[0]);
	
	while ((line = br.readLine()) != null) {
		parts = eatquotedStrings(line);
		for (i = 0; i < parts.length; i++) {
			parts[i] = parts[i].replaceAll("\"", "");
		}
		bloomFilter.put(parts[0]);
		this.size++;
	}
	br.close();
}
 
Example #19
Source File: SpillMap.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public MappedByteBufferMap(int id, int thresholdBytes, int pageInserts, SpillFile spillFile) {
    this.spillFile = spillFile;
    // size threshold of a page
    this.thresholdBytes = thresholdBytes;
    this.pageIndex = id;
    pageMap.clear();
    bFilter = BloomFilter.create(Funnels.byteArrayFunnel(), pageInserts);
    pagedIn = true;
    totalResultSize = 0;
    localDepth = 1;
    dirtyPage = true;
}
 
Example #20
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyPartFull() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	for (int i = 0; i < 1000000; i++) {
		assertTrue(filter.put(i));
	}
	CuckooFilter<Integer> filterCopy = filter.copy();
	assertTrue(filterCopy.equals(filter));
	assertNotSame(filter, filterCopy);
}
 
Example #21
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateDifferentHashLengths() {
	new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000).withHashAlgorithm(Algorithm.Murmur3_32).build();
	new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000).withHashAlgorithm(Algorithm.sipHash24).build();
	new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000).withHashAlgorithm(Algorithm.Murmur3_128).build();
	new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000).withHashAlgorithm(Algorithm.sha256).build();
}
 
Example #22
Source File: Cuckoo.java    From XRTB with Apache License 2.0 5 votes vote down vote up
/**
 * Reads a file or S3 object line by line and loads the filter.
 * @param br BufferedReader. The line-by-line reader.
 * @throws Exception on I/O errors.
 */
void makeFilter(BufferedReader br, long sz) throws Exception {
	String[] parts;
	int i;

	
	String line = br.readLine();
	line = line.trim();
	i = 0;
	
	parts = eatquotedStrings(line);
	for (i = 0; i < parts.length; i++) {
		parts[i] = parts[i].replaceAll("\"", "");
	}
	long size = parts[0].length() - 5;
	size = sz / size;
	double fpp = 0.03; // desired false positive probability
	cuckooFilter = new CuckooFilter.Builder<>(Funnels.stringFunnel(Charset.forName("UTF-8")), size).build();
	cuckooFilter.put(parts[0]);
	

	while ((line = br.readLine()) != null) {
		parts = eatquotedStrings(line);
		for (i = 0; i < parts.length; i++) {
			parts[i] = parts[i].replaceAll("\"", "");
		}
		cuckooFilter.put(parts[0]);
	}
	br.close();
}
 
Example #23
Source File: RendezvousHashUnitTest.java    From xio with Apache License 2.0 5 votes vote down vote up
@Test
public void get() throws Exception {
  List<String> nodeList = new ArrayList<>();
  Map<String, List<String>> mm = PlatformDependent.newConcurrentHashMap();
  for (int i = 0; i < 100; i++) {
    nodeList.add(("Host" + i));
    mm.put(("Host" + i), new ArrayList<>());
  }

  RendezvousHash<CharSequence> rendezvousHash =
      new RendezvousHash<>(Funnels.stringFunnel(Charset.defaultCharset()), nodeList);
  Random r = new Random();
  for (int i = 0; i < 100000; i++) {
    String thing = (Integer.toString(r.nextInt(123456789)));
    List<CharSequence> hosts = rendezvousHash.get(thing.getBytes(), 3);
    hosts.forEach(
        xs -> {
          mm.get(xs).add(thing);
        });
  }

  List<Integer> xx = new ArrayList<>();
  mm.keySet()
      .forEach(
          xs -> {
            xx.add(mm.get(xs).size());
          });

  Double xd = xx.stream().mapToInt(x -> x).average().orElse(-1);
  assertEquals(3000, xd.intValue());
}
 
Example #24
Source File: TestCuckooFilter.java    From CuckooFilter4J with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializeFull() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	for (int i = 0;; i++) {
		if (!filter.put(i))
			break;
	}
	SerializableTester.reserializeAndAssert(filter);
}
 
Example #25
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void createAndCheckBealDupras32CuckooFilterWithKnownFalsePositives() {
  int numInsertions = 1000000;
  CuckooFilter<String> cf = CuckooFilter.create(
      Funnels.unencodedCharsFunnel(), numInsertions, 0.03,
      CuckooStrategies.MURMUR128_BEALDUPRAS_32.strategy());

  // Insert "numInsertions" even numbers into the CF.
  for (int i = 0; i < numInsertions * 2; i += 2) {
    cf.add(Integer.toString(i));
  }

  // Assert that the CF "might" have all of the even numbers.
  for (int i = 0; i < numInsertions * 2; i += 2) {
    assertTrue(cf.contains(Integer.toString(i)));
  }

  // Now we check for known false positives using a set of known false positives.
  // (These are all of the false positives under 900.)
  ImmutableSet<Integer> falsePositives = ImmutableSet.of(217, 329, 581, 707, 757, 805, 863);
  for (int i = 1; i < 900; i += 2) {
    if (!falsePositives.contains(i)) {
      assertFalse("CF should not contain " + i, cf.contains(Integer.toString(i)));
    }
  }

  // Check that there are exactly 25926 false positives for this CF.
  int expectedNumFpp = 25926;
  int actualNumFpp = 0;
  for (int i = 1; i < numInsertions * 2; i += 2) {
    if (cf.contains(Integer.toString(i))) {
      actualNumFpp++;
    }
  }
  assertEquals(expectedNumFpp, actualNumFpp);
  // The normal order of (expected, actual) is reversed here on purpose.
  assertEquals((double) expectedNumFpp / numInsertions, cf.currentFpp(), 0.00035);
}
 
Example #26
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void createAndCheckBealDupras32CuckooFilterWithKnownUtf8FalsePositives() {
  int numInsertions = 1000000;
  CuckooFilter<String> cf = CuckooFilter.create(
      Funnels.stringFunnel(UTF_8), numInsertions, 0.03,
      CuckooStrategies.MURMUR128_BEALDUPRAS_32.strategy());

  // Insert "numInsertions" even numbers into the CF.
  for (int i = 0; i < numInsertions * 2; i += 2) {
    cf.add(Integer.toString(i));
  }

  // Assert that the CF "might" have all of the even numbers.
  for (int i = 0; i < numInsertions * 2; i += 2) {
    assertTrue(cf.contains(Integer.toString(i)));
  }

  // Now we check for known false positives using a set of known false positives.
  // (These are all of the false positives under 900.)
  ImmutableSet<Integer> falsePositives =
      ImmutableSet.of(5, 315, 389, 443, 445, 615, 621, 703, 789, 861, 899);
  for (int i = 1; i < 900; i += 2) {
    if (!falsePositives.contains(i)) {
      assertFalse("CF should not contain " + i, cf.contains(Integer.toString(i)));
    }
  }

  // Check that there are exactly 26610 false positives for this CF.
  int expectedNumFpp = 26610;
  int actualNumFpp = 0;
  for (int i = 1; i < numInsertions * 2; i += 2) {
    if (cf.contains(Integer.toString(i))) {
      actualNumFpp++;
    }
  }
  assertEquals(expectedNumFpp, actualNumFpp);
  // The normal order of (expected, actual) is reversed here on purpose.
  assertEquals((double) expectedNumFpp / numInsertions, cf.currentFpp(), 0.0004);
}
 
Example #27
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void failureWhenMoreThan64BitFingerprintsAreNeeded() {
  try {
    int n = 1000;
    double p = 0.00000000000000000000000000000000000000000000000000000000000000000000000000000001;
    CuckooFilter.create(Funnels.unencodedCharsFunnel(), n, p);
    fail();
  } catch (IllegalArgumentException expected) {
  }
}
 
Example #28
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void bitSize() {
  double fpp = 0.03;
  for (int i = 1; i < 10000; i++) {
    long numBits = CuckooFilter.calculateDataLength(i, fpp) * Long.SIZE;
    int arraySize = Ints.checkedCast(LongMath.divide(numBits, Long.SIZE, RoundingMode.CEILING));
    assertEquals(
        arraySize * Long.SIZE,
        CuckooFilter.create(Funnels.unencodedCharsFunnel(), i, fpp).bitSize());
  }
}
 
Example #29
Source File: CuckooFilterTest.java    From guava-probably with Apache License 2.0 5 votes vote down vote up
@Test
public void equals_empty() {
  new EqualsTester()
      .addEqualityGroup(CuckooFilter.create(Funnels.byteArrayFunnel(), 100, 0.01))
      .addEqualityGroup(CuckooFilter.create(Funnels.byteArrayFunnel(), 100, 0.02))
      .addEqualityGroup(CuckooFilter.create(Funnels.byteArrayFunnel(), 200, 0.01))
      .addEqualityGroup(CuckooFilter.create(Funnels.byteArrayFunnel(), 200, 0.02))
      .addEqualityGroup(CuckooFilter.create(Funnels.unencodedCharsFunnel(), 100, 0.01))
      .addEqualityGroup(CuckooFilter.create(Funnels.unencodedCharsFunnel(), 100, 0.02))
      .addEqualityGroup(CuckooFilter.create(Funnels.unencodedCharsFunnel(), 200, 0.01))
      .addEqualityGroup(CuckooFilter.create(Funnels.unencodedCharsFunnel(), 200, 0.02))
      .testEquals();
}
 
Example #30
Source File: RendezvousHashTest.java    From xrpc with Apache License 2.0 5 votes vote down vote up
@Test
void simpleGet() {
  Map<String, String> map =
      new ImmutableMap.Builder<String, String>()
          .put("a", "1")
          .put("b", "2")
          .put("c", "3")
          .put("d", "4")
          .put("e", "5")
          .build();
  RendezvousHash<CharSequence> hasher =
      new RendezvousHash<>(Funnels.stringFunnel(XrpcConstants.DEFAULT_CHARSET), map.keySet());
  String k1 = "foo";
  String k2 = "bar";
  String k3 = "baz";

  assertEquals(hasher.getOne(k1.getBytes()), hasher.getOne(k1.getBytes()));
  assertEquals(hasher.getOne(k2.getBytes()), hasher.getOne(k2.getBytes()));
  assertEquals(hasher.getOne(k3.getBytes()), hasher.getOne(k3.getBytes()));
  String k4 = "biz";
  assertEquals(hasher.getOne(k4.getBytes()), hasher.getOne(k4.getBytes()));

  System.out.println(hasher.getOne(k1.getBytes()));
  System.out.println(hasher.getOne(k2.getBytes()));
  System.out.println(hasher.getOne(k3.getBytes()));
  System.out.println(hasher.getOne(k4.getBytes()));

  System.out.println(hasher.getOne(k1.getBytes()));
  System.out.println(hasher.getOne(k2.getBytes()));
  System.out.println(hasher.getOne(k3.getBytes()));
  System.out.println(hasher.getOne(k4.getBytes()));

  assertNotEquals(hasher.getOne(k1.getBytes()), hasher.getOne(k4.getBytes()));
}