com.google.common.hash.Funnels Java Examples

The following examples show how to use com.google.common.hash.Funnels. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: n4js   Author: eclipse   File: HashedFileContent.java    License: Eclipse Public License 1.0 6 votes vote down vote up
/** Create a fingerprint of the given file at the given location. */
public HashedFileContent(URI uri, File file) throws IOException {
	this.uri = uri;
	String ext = uri.fileExtension();
	if (ext == null || "ts".equals(ext) || "js".equals(ext) || "jsx".equals(ext) || "map".equals(ext)
			|| "md".equals(ext)
			|| "hbs".equals(ext)
			|| "json".equals(ext) && !"package.json".equals(uri.lastSegment())) {
		this.hash = file.length();
	} else {
		try (InputStream s = new FileInputStream(file)) {
			Hasher hasher = hashFunction.newHasher();
			s.transferTo(Funnels.asOutputStream(hasher));
			this.hash = hasher.hash().asLong();
		}
	}
}
 
Example #2
Source Project: keycloak   Author: keycloak   File: BlacklistPasswordPolicyProviderFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Loads the referenced blacklist into a {@link BloomFilter}.
 *
 * @return the {@link BloomFilter} backing a password blacklist
 */
private BloomFilter<String> load() {

    try {
        LOG.infof("Loading blacklist with name %s from %s - start", name, path);

        long passwordCount = getPasswordCount();

        BloomFilter<String> filter = BloomFilter.create(
                Funnels.stringFunnel(StandardCharsets.UTF_8),
                passwordCount,
                FALSE_POSITIVE_PROBABILITY);

        try (BufferedReader br = newReader(path)) {
            br.lines().forEach(filter::put);
        }

        LOG.infof("Loading blacklist with name %s from %s - end", name, path);

        return filter;
    } catch (IOException e) {
        throw new RuntimeException("Could not load password blacklist from path: " + path, e);
    }
}
 
Example #3
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestIndexTagCalc.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void knownZeroTags32() {
	// manual instantiation to force salts to be static
	SerializableSaltedHasher<Integer> hasher = new SerializableSaltedHasher<>(0, 0, Funnels.integerFunnel(),
			Algorithm.Murmur3_32);
	IndexTagCalc<Integer> indexer = new IndexTagCalc<>(hasher, 128, 4);
	// find some 0 value tags
	int zeroTags = 0;
	int i = 0;
	ArrayList<Integer> zeroTagInputs = new ArrayList<>();
	while (zeroTags < 20) {
		if (indexer.getTagValue32(hasher.hashObj(i).asInt()) == 0) {
			zeroTagInputs.add(i);
			zeroTags++;
		}
		i++;
	}
	for (Integer tag : zeroTagInputs) {
		// none of the zero value tags should be zero from indexer if
		// rehashing is working properly
		assertFalse(indexer.generate(tag).tag == 0);
	}

}
 
Example #4
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestIndexTagCalc.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void sanityTagIndexBitsUsed64() {
	// manual instantiation to force salts to be static
	SerializableSaltedHasher<Integer> hasher = new SerializableSaltedHasher<>(0, 0, Funnels.integerFunnel(),
			Algorithm.sipHash24);
	IndexTagCalc<Integer> indexer = new IndexTagCalc<>(hasher, (long) Math.pow(2, 31), 32);
	long setBitsIndex = 0;
	long setBitsTag = 0;
	// should be enough to set all bits being used...
	for (int i = 0; i < 1234567; i++) {
		BucketAndTag bt = indexer.generate(i);
		setBitsIndex |= bt.index;
		setBitsTag |= bt.tag;
	}
	// will be true if we're using the right number of bits for tag and
	// index for this calculator
	assertTrue(Long.bitCount(setBitsIndex) == 31);
	assertTrue(Long.bitCount(setBitsTag) == 32);
	// check where the set bits are
	long bitMask32 = -1L >>> 32;// (mask for lower 32 bits set)
	long bitMask31 = bitMask32 >>> 1;// (mask for lower 32 bits set)
	assertTrue(bitMask32 == setBitsTag);
	assertTrue(bitMask31 == setBitsIndex);
}
 
Example #5
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestCuckooFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFalseNegative() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	// add them to filter
	for (int i = 0; i < 100000; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false negatives
	int falseNegatives = 0;
	for (int i = 0; i < 100000; i++) {
		if (!filter.mightContain(i)) {
			falseNegatives++;
		}
	}
	assertTrue(falseNegatives + " false negatives detected", falseNegatives == 0);

}
 
Example #6
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestCuckooFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFailedDelete() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();

	// make a list of test values(all unique)
	int maxInsertedVal = 100000;
	for (int i = 0; i < maxInsertedVal; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false deletes(if I can't delete something that's definitely
	// there)
	int falseDeletes = 0;
	for (int i = 0; i < maxInsertedVal; i++) {
		if (!filter.delete(i)) {
			falseDeletes++;
		}
	}
	assertTrue(falseDeletes + " false deletions detected", falseDeletes == 0);
}
 
Example #7
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestCuckooFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFalseDeleteRate() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	int maxInsertedVal = 100000;
	for (int i = 0; i < maxInsertedVal; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false delete rate(deleted something I didn't add
	// successfully)
	int falseDeletes = 0;
	// false delete rate should roughly match false positive rate
	int totalAttempts = 10000;
	maxInsertedVal += 1;
	for (int i = maxInsertedVal; i < totalAttempts + maxInsertedVal; i++) {
		if (filter.delete(i))
			falseDeletes++;
	}
	assertTrue(
			falseDeletes
					+ " false deletions detected. False delete rate is above 0.02 on filter with 0.01 false positive rate",
			(double) falseDeletes / totalAttempts < 0.02);

}
 
Example #8
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestCuckooFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void sanityFalsePositiveRate() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	int maxInsertedVal = 100000;
	// make a list of test values(all unique)
	for (int i = 0; i < maxInsertedVal; i++) {
		// will return false if filter is full...should NOT be
		assertTrue(filter.put(i));
	}
	// check for false positive rate(contains something I didn't add)
	int falsePositives = 0;
	maxInsertedVal += 1;
	int totalAttempts = 100000;
	for (int i = maxInsertedVal; i < totalAttempts + maxInsertedVal; i++) {
		if (filter.mightContain(i))
			falsePositives++;
	}
	assertTrue((double) falsePositives / totalAttempts + " false positive rate is above limit",
			(double) falsePositives / totalAttempts < 0.02);

}
 
Example #9
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestCuckooFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void sanityTestVictimCache() {
	CuckooFilter<Integer> filter = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 130000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();

	for (int i = 0; i < 9; i++) {
		assertTrue(filter.put(42));
	}
	assertTrue(filter.getCount() == 9);
	for (int i = 0; i < 9; i++) {
		assertTrue(filter.mightContain(42));
		assertTrue(filter.delete(42));
	}
	assertFalse(filter.delete(42));
	assertFalse(filter.mightContain(42));
	assertTrue(filter.getCount() == 0);
	// at this point victim cache is in use since both buckets for 42 are
	// full

}
 
Example #10
Source Project: tutorials   Author: eugenp   File: BloomFilterUnitTest.java    License: MIT License 6 votes vote down vote up
@Test
public void givenBloomFilter_whenAddNStringsToIt_thenShouldNotReturnAnyFalsePositive() {
    //when
    BloomFilter<Integer> filter = BloomFilter.create(
            Funnels.integerFunnel(),
            500,
            0.01);

    //when
    filter.put(1);
    filter.put(2);
    filter.put(3);

    //then
    // the probability that it returns true, but is actually false is 1%
    assertThat(filter.mightContain(1)).isTrue();
    assertThat(filter.mightContain(2)).isTrue();
    assertThat(filter.mightContain(3)).isTrue();

    assertThat(filter.mightContain(100)).isFalse();
}
 
Example #11
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestCuckooFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEquals() {
	CuckooFilter<Integer> partFull = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	CuckooFilter<Integer> full = new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
			.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build();
	for (int i = 0; i < 1000000; i++) {
		assertTrue(partFull.put(i));
	}
	for (int i = 0;; i++) {
		if (!full.put(i))
			break;
	}
	new EqualsTester().addEqualityGroup(partFull).addEqualityGroup(full)
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
					.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.longFunnel(), 2000000).withFalsePositiveRate(0.01)
					.withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 1000000)
					.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
					.withFalsePositiveRate(0.03).withHashAlgorithm(Algorithm.Murmur3_32).build())
			.addEqualityGroup(new CuckooFilter.Builder<>(Funnels.integerFunnel(), 2000000)
					.withFalsePositiveRate(0.01).withHashAlgorithm(Algorithm.Murmur3_128).build())
			.testEquals();
}
 
Example #12
Source Project: guava-probably   Author: bdupras   File: CuckooFilterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void addAll() {
  int element1 = 1;
  int element2 = 2;

  CuckooFilter<Integer> cf1 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf1.add(element1);
  assertTrue(cf1.contains(element1));
  assertFalse(cf1.contains(element2));

  CuckooFilter<Integer> cf2 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf2.add(element2);
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));

  assertTrue(cf1.isCompatible(cf2));
  cf1.addAll(cf2);
  assertTrue(cf1.contains(element1));
  assertTrue(cf1.contains(element2));
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));
}
 
Example #13
Source Project: runelite   Author: runelite   File: XpTrackerService.java    License: BSD 2-Clause "Simplified" License 6 votes vote down vote up
private BloomFilter<String> createFilter()
{
	final BloomFilter<String> filter = BloomFilter.create(
		Funnels.stringFunnel(Charset.defaultCharset()),
		BLOOMFILTER_EXPECTED_INSERTIONS
	);

	synchronized (usernameUpdateQueue)
	{
		for (String toUpdate : usernameUpdateQueue)
		{
			filter.put(toUpdate);
		}
	}

	return filter;
}
 
Example #14
Source Project: guava-probably   Author: bdupras   File: CuckooFilterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void addAll() {
  int element1 = 1;
  int element2 = 2;

  CuckooFilter<Integer> cf1 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf1.add(element1);
  assertTrue(cf1.contains(element1));
  assertFalse(cf1.contains(element2));

  CuckooFilter<Integer> cf2 = CuckooFilter.create(Funnels.integerFunnel(), 100);
  cf2.add(element2);
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));

  assertTrue(cf1.isCompatible(cf2));
  cf1.addAll(cf2);
  assertTrue(cf1.contains(element1));
  assertTrue(cf1.contains(element2));
  assertFalse(cf2.contains(element1));
  assertTrue(cf2.contains(element2));
}
 
Example #15
Source Project: bidder   Author: RTB4FREE   File: Cuckoo.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads a file or S3 object line by line and loads the filter.
 * @param br BufferedReader. The line-by-line reader.
 * @throws Exception on I/O errors.
 */
void makeFilter(BufferedReader br, long sz) throws Exception {
	String[] parts;
	int i;

	
	String line = br.readLine();
	line = line.trim();
	i = 0;
	
	parts = eatquotedStrings(line);
	for (i = 0; i < parts.length; i++) {
		parts[i] = parts[i].replaceAll("\"", "");
	}
	long size = parts[0].length() - 5;
	size = sz / size;
	double fpp = 0.03; // desired false positive probability
	cuckooFilter = new CuckooFilter.Builder<>(Funnels.stringFunnel(Charset.forName("UTF-8")), size).build();
	cuckooFilter.put(parts[0]);
	

	while ((line = br.readLine()) != null) {
		parts = eatquotedStrings(line);
		for (i = 0; i < parts.length; i++) {
			parts[i] = parts[i].replaceAll("\"", "");
		}
		cuckooFilter.put(parts[0]);
	}
	br.close();
}
 
Example #16
Source Project: phoenix   Author: cloudera-labs   File: SpillMap.java    License: Apache License 2.0 5 votes vote down vote up
public MappedByteBufferMap(int id, int thresholdBytes, int pageInserts, SpillFile spillFile) {
    this.spillFile = spillFile;
    // size threshold of a page
    this.thresholdBytes = thresholdBytes;
    this.pageIndex = id;
    pageMap.clear();
    bFilter = BloomFilter.create(Funnels.byteArrayFunnel(), pageInserts);
    pagedIn = true;
    totalResultSize = 0;
    localDepth = 1;
    dirtyPage = true;
}
 
Example #17
Source Project: bidder   Author: RTB4FREE   File: Bloom.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads a file or S3 object line by line and loads the filter.
 * @param br BufferedReader. The line-by-line reader.
 * @throws Exception on I/O errors.
 */
void makeFilter(BufferedReader br, long size) throws Exception {
	String[] parts;
	int i;
	long sz;
	
	double fpp = 0.003; // desired false positive probability
	
	String line = br.readLine();
	sz = line.length() - 5;
	if (sz < 0)
		sz = 16;                // just a guess
	sz = size / sz;
	sz *= 2;
	parts = eatquotedStrings(line);
	this.size = 1;
	for (i = 0; i < parts.length; i++) {
		parts[i] = parts[i].replaceAll("\"", "");
	}
	
	bloomFilter = BloomFilter.create(Funnels.stringFunnel(Charset.forName("UTF-8")), sz,fpp);
	bloomFilter.put(parts[0]);
	
	while ((line = br.readLine()) != null) {
		parts = eatquotedStrings(line);
		for (i = 0; i < parts.length; i++) {
			parts[i] = parts[i].replaceAll("\"", "");
		}
		bloomFilter.put(parts[0]);
		this.size++;
	}
	br.close();
}
 
Example #18
Source Project: codebuff   Author: antlr   File: ByteSource.java    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * Hashes the contents of this byte source using the given hash function.
 *
 * @throws IOException if an I/O error occurs in the process of reading from this source
 */


public HashCode hash(HashFunction hashFunction) throws IOException {
  Hasher hasher = hashFunction.newHasher();
  copyTo(Funnels.asOutputStream(hasher));
  return hasher.hash();
}
 
Example #19
Source Project: quarantyne   Author: quarantyne   File: CompromisedPasswordClassifierTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testClassifier() {
  BloomFilter<String> bloom =
      BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 3);
  bloom.put("alpha");
  bloom.put("bravo");
  bloom.put("charlie");

  Supplier<Config> configSupplier = () -> Config.builder()
      .loginAction(new QIdentityAction("/login", "email", "password"))
      .registerAction(new QIdentityAction("/register", "email", "password"))
      .build();
  CompromisedPasswordClassifier classifier = new CompromisedPasswordClassifier(bloom, configSupplier);
  HttpRequest defaultRequest = TestHttpRequest.REQ();

  // null empty
  assertThat(classifier.classify(defaultRequest, null)).isEqualTo(Label.NONE);
  assertThat(classifier.classify(defaultRequest, TestHttpRequestBody.EMPTY)).isEqualTo(Label.NONE);

  // no key matches password
  assertThat(classifier.classify(defaultRequest,
      TestHttpRequestBody.make(new JsonObject().put("name", "john")))).isEqualTo(Label.NONE);

  // a key matches password but password is not in bloomf
  assertThat(classifier.classify(defaultRequest,
      TestHttpRequestBody.make(new JsonObject().put("password", "delta")))).isEqualTo(Label.NONE);

  // match
  HttpRequest requestOnPath = new TestHttpRequest.Builder().setPath("/login").build();
  assertThat(classifier.classify(requestOnPath,
      TestHttpRequestBody.make(new JsonObject().put("password", "bravo")))).isEqualTo(
          Label.COMPROMISED_PASSWORD);
}
 
Example #20
Source Project: quarantyne   Author: quarantyne   File: DisposableEmailClassifierTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testClassifier() {
  BloomFilter<String> bloom =
      BloomFilter.create(Funnels.stringFunnel(Charsets.UTF_8), 2);
  bloom.put("disposable.com");
  bloom.put("junk.com");

  Supplier<Config> configSupplier = () -> Config
      .builder()
      .emailParamKeys(Sets.newHashSet("email"))
      .registerAction(new QIdentityAction("/register", "email", "password"))
      .build();
  DisposableEmailClassifier classifier = new DisposableEmailClassifier(bloom, configSupplier);
  HttpRequest defaultRequest = TestHttpRequest.REQ();

  // null empty
  assertThat(classifier.classify(defaultRequest, null)).isEqualTo(Label.NONE);
  assertThat(classifier.classify(defaultRequest, TestHttpRequestBody.EMPTY)).isEqualTo(Label.NONE);

  // no key matches password
  assertThat(classifier.classify(defaultRequest,
      TestHttpRequestBody.make(new JsonObject().put("name", "john")))).isEqualTo(Label.NONE);

  // a key matches password but password is not in bloomf
  assertThat(classifier.classify(defaultRequest,
      TestHttpRequestBody.make(new JsonObject().put("email", "[email protected]")))).isEqualTo(Label.NONE);

  // match
  HttpRequest req = new TestHttpRequest.Builder().setPath("/register").build();
  assertThat(classifier.classify(req,
      TestHttpRequestBody.make(new JsonObject().put("email", "[email protected]")))).isEqualTo(
      Label.DISPOSABLE_EMAIL);
}
 
Example #21
Source Project: xio   Author: xjdr   File: RendezvousHashUnitTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void simpleGet() {
  Map<String, String> map =
      new ImmutableMap.Builder<String, String>()
          .put("a", "1")
          .put("b", "2")
          .put("c", "3")
          .put("d", "4")
          .put("e", "5")
          .build();
  RendezvousHash<CharSequence> hasher =
      new RendezvousHash<>(Funnels.stringFunnel(Constants.DEFAULT_CHARSET), map.keySet());
  String k1 = "foo";
  String k2 = "bar";
  String k3 = "baz";

  assertEquals(hasher.getOne(k1.getBytes()), hasher.getOne(k1.getBytes()));
  assertEquals(hasher.getOne(k2.getBytes()), hasher.getOne(k2.getBytes()));
  assertEquals(hasher.getOne(k3.getBytes()), hasher.getOne(k3.getBytes()));
  String k4 = "biz";
  assertEquals(hasher.getOne(k4.getBytes()), hasher.getOne(k4.getBytes()));

  System.out.println(hasher.getOne(k1.getBytes()));
  System.out.println(hasher.getOne(k2.getBytes()));
  System.out.println(hasher.getOne(k3.getBytes()));
  System.out.println(hasher.getOne(k4.getBytes()));

  System.out.println(hasher.getOne(k1.getBytes()));
  System.out.println(hasher.getOne(k2.getBytes()));
  System.out.println(hasher.getOne(k3.getBytes()));
  System.out.println(hasher.getOne(k4.getBytes()));

  assertNotEquals(hasher.getOne(k1.getBytes()), hasher.getOne(k4.getBytes()));
}
 
Example #22
Source Project: phoenix   Author: apache   File: SpillMap.java    License: Apache License 2.0 5 votes vote down vote up
public FileMap(int id, int thresholdBytes, int pageInserts, SpillFile spillFile) {
    this.spillFile = spillFile;
    // size threshold of a page
    this.thresholdBytes = thresholdBytes;
    this.pageIndex = id;
    pageMap.clear();
    bFilter = BloomFilter.create(Funnels.byteArrayFunnel(), pageInserts);
    pagedIn = true;
    totalResultSize = 0;
    localDepth = 1;
    dirtyPage = true;
}
 
Example #23
Source Project: xrpc   Author: Nordstrom   File: ServiceRateLimiter.java    License: Apache License 2.0 5 votes vote down vote up
private RendezvousHash<CharSequence> buildHasher(
    Map<String, RateLimiter> limiterMap, int poolSize, double rate) {
  List<String> tempPool = new ArrayList<>();

  for (int i = 0; i < poolSize; i++) {
    String id = UUID.randomUUID().toString();
    tempPool.add(id);
    limiterMap.put(id, RateLimiter.create(rate));
  }

  return new RendezvousHash<>(Funnels.stringFunnel(XrpcConstants.DEFAULT_CHARSET), tempPool);
}
 
Example #24
Source Project: xrpc   Author: Nordstrom   File: RendezvousHashTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void get_shouldReturnExpectedNumberOfHashesOnAllRuns() throws Exception {
  List<String> hostList = new ArrayList<>();
  Map<String, List<String>> hostToMatchingHashes = PlatformDependent.newConcurrentHashMap();
  int totalHosts = 100;
  for (int i = 0; i < totalHosts; i++) {
    hostList.add(("Host" + i));
    hostToMatchingHashes.put(("Host" + i), new ArrayList<>());
  }

  RendezvousHash<CharSequence> rendezvousHash =
      new RendezvousHash<>(Funnels.stringFunnel(Charset.defaultCharset()), hostList);

  int totalGetsToRun = 100000;
  int hashesToMatch = 3;
  Random random = new Random();
  for (int i = 0; i < totalGetsToRun; i++) {
    String randomNumberString = (Integer.toString(random.nextInt(MAX_RANDOM_NUMBER)));
    List<CharSequence> hosts = rendezvousHash.get(randomNumberString.getBytes(), hashesToMatch);
    hosts.forEach(host -> hostToMatchingHashes.get(host).add(randomNumberString));
  }

  Double averageMatchingHashesPerHost =
      hostToMatchingHashes.values().stream().mapToInt(List::size).average().orElse(-1);

  int expectedAverage = hashesToMatch * totalGetsToRun / totalHosts;
  assertEquals(expectedAverage, averageMatchingHashesPerHost.intValue());
}
 
Example #25
Source Project: xrpc   Author: Nordstrom   File: RendezvousHashTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
void simpleGet() {
  Map<String, String> map =
      new ImmutableMap.Builder<String, String>()
          .put("a", "1")
          .put("b", "2")
          .put("c", "3")
          .put("d", "4")
          .put("e", "5")
          .build();
  RendezvousHash<CharSequence> hasher =
      new RendezvousHash<>(Funnels.stringFunnel(XrpcConstants.DEFAULT_CHARSET), map.keySet());
  String k1 = "foo";
  String k2 = "bar";
  String k3 = "baz";

  assertEquals(hasher.getOne(k1.getBytes()), hasher.getOne(k1.getBytes()));
  assertEquals(hasher.getOne(k2.getBytes()), hasher.getOne(k2.getBytes()));
  assertEquals(hasher.getOne(k3.getBytes()), hasher.getOne(k3.getBytes()));
  String k4 = "biz";
  assertEquals(hasher.getOne(k4.getBytes()), hasher.getOne(k4.getBytes()));

  System.out.println(hasher.getOne(k1.getBytes()));
  System.out.println(hasher.getOne(k2.getBytes()));
  System.out.println(hasher.getOne(k3.getBytes()));
  System.out.println(hasher.getOne(k4.getBytes()));

  System.out.println(hasher.getOne(k1.getBytes()));
  System.out.println(hasher.getOne(k2.getBytes()));
  System.out.println(hasher.getOne(k3.getBytes()));
  System.out.println(hasher.getOne(k4.getBytes()));

  assertNotEquals(hasher.getOne(k1.getBytes()), hasher.getOne(k4.getBytes()));
}
 
Example #26
Source Project: guava-probably   Author: bdupras   File: CuckooFilterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void addAllWithSelf() {
  CuckooFilter<Integer> cf1 = CuckooFilter.create(Funnels.integerFunnel(), 1);
  try {
    assertFalse(cf1.isCompatible(cf1));
    cf1.addAll(cf1);
    fail();
  } catch (IllegalArgumentException expected) {
  }
}
 
Example #27
Source Project: guava-probably   Author: bdupras   File: CuckooFilterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void failureWhenMoreThan64BitFingerprintsAreNeeded() {
  try {
    int n = 1000;
    double p = 0.00000000000000000000000000000000000000000000000000000000000000000000000000000001;
    CuckooFilter.create(Funnels.unencodedCharsFunnel(), n, p);
    fail();
  } catch (IllegalArgumentException expected) {
  }
}
 
Example #28
Source Project: bazel-buildfarm   Author: bazelbuild   File: DigestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public HashCode computeHash(ByteString blob) {
  Hasher hasher = hashFn.getHash().newHasher();
  try {
    blob.writeTo(Funnels.asOutputStream(hasher));
  } catch (IOException e) {
    /* impossible, due to Funnels.asOutputStream behavior */
  }
  return hasher.hash();
}
 
Example #29
Source Project: nexus-public   Author: sonatype   File: Hashes.java    License: Eclipse Public License 1.0 5 votes vote down vote up
/**
 * Computes the hash of the given stream using the given function.
 */
public static HashCode hash(final HashFunction function, final InputStream input) throws IOException {
  Hasher hasher = function.newHasher();
  OutputStream output = Funnels.asOutputStream(hasher);
  ByteStreams.copy(input, output);
  return hasher.hash();
}
 
Example #30
Source Project: CuckooFilter4J   Author: MGunlogson   File: TestSerializableSaltedHasher.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEquals() {
	new EqualsTester()
			.addEqualityGroup(
					new SerializableSaltedHasher<byte[]>(0, 0, Funnels.byteArrayFunnel(), Algorithm.Murmur3_32))
			.addEqualityGroup(
					new SerializableSaltedHasher<byte[]>(1, 0, Funnels.byteArrayFunnel(), Algorithm.Murmur3_32))
			.addEqualityGroup(
					new SerializableSaltedHasher<byte[]>(0, 1, Funnels.byteArrayFunnel(), Algorithm.Murmur3_32))
			.addEqualityGroup(
					new SerializableSaltedHasher<Integer>(0, 0, Funnels.integerFunnel(), Algorithm.Murmur3_32))
			.addEqualityGroup(
					new SerializableSaltedHasher<byte[]>(0, 0, Funnels.byteArrayFunnel(), Algorithm.sha256))
			.testEquals();
}