Java Code Examples for org.apache.hadoop.util.hash.Hash#JENKINS_HASH

The following examples show how to use org.apache.hadoop.util.hash.Hash#JENKINS_HASH . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestBloomFilters.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testDynamicBloomFilter() {
  int hashId = Hash.JENKINS_HASH;    
  Filter filter = new DynamicBloomFilter(bitSize, hashFunctionNumber,
      Hash.JENKINS_HASH, 3);    
  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY))
              .test();
  
  assertNotNull("testDynamicBloomFilter error ", filter.toString());
}
 
Example 2
Source File: TestBloomFilters.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiltersWithJenkinsHash() {
  int hashId = Hash.JENKINS_HASH;

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(new BloomFilter(bitSize, hashFunctionNumber, hashId))
      .withFilterInstance(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId))
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_AND_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}
 
Example 3
Source File: BloomFilterCommonTester.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private BloomFilterCommonTester(int hashId, int numInsertions) {
  this.hashType = hashId;
  this.numInsertions = numInsertions;

  this.preAssertionHelper = new PreAssertionHelper() {

    @Override
    public ImmutableSet<Integer> falsePositives(int hashId) {
      switch (hashId) {
      case Hash.JENKINS_HASH: {
        // // false pos for odd and event under 1000
        return ImmutableSet.of(99, 963);
      }
      case Hash.MURMUR_HASH: {
        // false pos for odd and event under 1000
        return ImmutableSet.of(769, 772, 810, 874);
      }
      default: {
        // fail fast with unknown hash error !!!
        Assert.assertFalse("unknown hash error", true);
        return ImmutableSet.of();
      }
      }
    }
  };
}
 
Example 4
Source File: TestBloomFilters.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testDynamicBloomFilter() {
  int hashId = Hash.JENKINS_HASH;    
  Filter filter = new DynamicBloomFilter(bitSize, hashFunctionNumber,
      Hash.JENKINS_HASH, 3);    
  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY))
              .test();
  
  assertNotNull("testDynamicBloomFilter error ", filter.toString());
}
 
Example 5
Source File: TestBloomFilters.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFiltersWithJenkinsHash() {
  int hashId = Hash.JENKINS_HASH;

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(new BloomFilter(bitSize, hashFunctionNumber, hashId))
      .withFilterInstance(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId))
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_AND_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}
 
Example 6
Source File: BloomFilterCommonTester.java    From big-c with Apache License 2.0 6 votes vote down vote up
private BloomFilterCommonTester(int hashId, int numInsertions) {
  this.hashType = hashId;
  this.numInsertions = numInsertions;

  this.preAssertionHelper = new PreAssertionHelper() {

    @Override
    public ImmutableSet<Integer> falsePositives(int hashId) {
      switch (hashId) {
      case Hash.JENKINS_HASH: {
        // // false pos for odd and event under 1000
        return ImmutableSet.of(99, 963);
      }
      case Hash.MURMUR_HASH: {
        // false pos for odd and event under 1000
        return ImmutableSet.of(769, 772, 810, 874);
      }
      default: {
        // fail fast with unknown hash error !!!
        Assert.assertFalse("unknown hash error", true);
        return ImmutableSet.of();
      }
      }
    }
  };
}
 
Example 7
Source File: Filter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}
 
Example 8
Source File: TestBloomFilters.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testCountingBloomFilter() {
  int hashId = Hash.JENKINS_HASH;

  CountingBloomFilter filter = new CountingBloomFilter(bitSize,
      hashFunctionNumber, hashId);

  Key key = new Key(new byte[] { 48, 48 });

  filter.add(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 1);

  filter.add(key);
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 2);

  filter.delete(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));

  filter.delete(key);
  assertFalse("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 0);

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}
 
Example 9
Source File: TestBloomFilters.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testNot() {
  BloomFilter bf = new BloomFilter(8, 1, Hash.JENKINS_HASH);
  bf.bits = BitSet.valueOf(new byte[] { (byte) 0x95 });
  BitSet origBitSet = (BitSet) bf.bits.clone();
  bf.not();
  assertFalse("BloomFilter#not should have inverted all bits",
              bf.bits.intersects(origBitSet));
}
 
Example 10
Source File: Filter.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}
 
Example 11
Source File: TestBloomFilters.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testCountingBloomFilter() {
  int hashId = Hash.JENKINS_HASH;

  CountingBloomFilter filter = new CountingBloomFilter(bitSize,
      hashFunctionNumber, hashId);

  Key key = new Key(new byte[] { 48, 48 });

  filter.add(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 1);

  filter.add(key);
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 2);

  filter.delete(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));

  filter.delete(key);
  assertFalse("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 0);

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}
 
Example 12
Source File: TestBloomFilters.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testNot() {
  BloomFilter bf = new BloomFilter(8, 1, Hash.JENKINS_HASH);
  bf.bits = BitSet.valueOf(new byte[] { (byte) 0x95 });
  BitSet origBitSet = (BitSet) bf.bits.clone();
  bf.not();
  assertFalse("BloomFilter#not should have inverted all bits",
              bf.bits.intersects(origBitSet));
}
 
Example 13
Source File: InternalFilter.java    From hudi with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}
 
Example 14
Source File: BuildBloomBase.java    From spork with Apache License 2.0 5 votes vote down vote up
private int convertHashType(String hashType) {
    if (hashType.toLowerCase().contains("jenkins")) {
        return Hash.JENKINS_HASH;
    } else if (hashType.toLowerCase().contains("murmur")) {
        return Hash.MURMUR_HASH;
    } else {
        throw new RuntimeException("Unknown hash type " + hashType +
            ".  Valid values are jenkins and murmur.");
    }
}
 
Example 15
Source File: Filter.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}
 
Example 16
Source File: Filter.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}