Java Code Examples for org.apache.hadoop.util.hash.Hash#JENKINS_HASH

The following examples show how to use org.apache.hadoop.util.hash.Hash#JENKINS_HASH . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestBloomFilters.java From hadoop with Apache License 2.0

6 votes

@Test
public void testDynamicBloomFilter() {
  int hashId = Hash.JENKINS_HASH;    
  Filter filter = new DynamicBloomFilter(bitSize, hashFunctionNumber,
      Hash.JENKINS_HASH, 3);    
  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY))
              .test();
  
  assertNotNull("testDynamicBloomFilter error ", filter.toString());
}

Example 2

Source File: TestBloomFilters.java From hadoop with Apache License 2.0

6 votes

@Test
public void testFiltersWithJenkinsHash() {
  int hashId = Hash.JENKINS_HASH;

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(new BloomFilter(bitSize, hashFunctionNumber, hashId))
      .withFilterInstance(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId))
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_AND_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}

Example 3

Source File: BloomFilterCommonTester.java From hadoop with Apache License 2.0

6 votes

private BloomFilterCommonTester(int hashId, int numInsertions) {
  this.hashType = hashId;
  this.numInsertions = numInsertions;

  this.preAssertionHelper = new PreAssertionHelper() {

    @Override
    public ImmutableSet<Integer> falsePositives(int hashId) {
      switch (hashId) {
      case Hash.JENKINS_HASH: {
        // // false pos for odd and event under 1000
        return ImmutableSet.of(99, 963);
      }
      case Hash.MURMUR_HASH: {
        // false pos for odd and event under 1000
        return ImmutableSet.of(769, 772, 810, 874);
      }
      default: {
        // fail fast with unknown hash error !!!
        Assert.assertFalse("unknown hash error", true);
        return ImmutableSet.of();
      }
      }
    }
  };
}

Example 4

Source File: TestBloomFilters.java From big-c with Apache License 2.0

6 votes

@Test
public void testDynamicBloomFilter() {
  int hashId = Hash.JENKINS_HASH;    
  Filter filter = new DynamicBloomFilter(bitSize, hashFunctionNumber,
      Hash.JENKINS_HASH, 3);    
  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY))
              .test();
  
  assertNotNull("testDynamicBloomFilter error ", filter.toString());
}

Example 5

Source File: TestBloomFilters.java From big-c with Apache License 2.0

6 votes

@Test
public void testFiltersWithJenkinsHash() {
  int hashId = Hash.JENKINS_HASH;

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(new BloomFilter(bitSize, hashFunctionNumber, hashId))
      .withFilterInstance(new RetouchedBloomFilter(bitSize, hashFunctionNumber, hashId))
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_AND_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}

Example 6

Source File: BloomFilterCommonTester.java From big-c with Apache License 2.0

6 votes

private BloomFilterCommonTester(int hashId, int numInsertions) {
  this.hashType = hashId;
  this.numInsertions = numInsertions;

  this.preAssertionHelper = new PreAssertionHelper() {

    @Override
    public ImmutableSet<Integer> falsePositives(int hashId) {
      switch (hashId) {
      case Hash.JENKINS_HASH: {
        // // false pos for odd and event under 1000
        return ImmutableSet.of(99, 963);
      }
      case Hash.MURMUR_HASH: {
        // false pos for odd and event under 1000
        return ImmutableSet.of(769, 772, 810, 874);
      }
      default: {
        // fail fast with unknown hash error !!!
        Assert.assertFalse("unknown hash error", true);
        return ImmutableSet.of();
      }
      }
    }
  };
}

Example 7

Source File: Filter.java From hadoop with Apache License 2.0

5 votes

@Override
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}

Example 8

Source File: TestBloomFilters.java From hadoop with Apache License 2.0

5 votes

@Test
public void testCountingBloomFilter() {
  int hashId = Hash.JENKINS_HASH;

  CountingBloomFilter filter = new CountingBloomFilter(bitSize,
      hashFunctionNumber, hashId);

  Key key = new Key(new byte[] { 48, 48 });

  filter.add(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 1);

  filter.add(key);
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 2);

  filter.delete(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));

  filter.delete(key);
  assertFalse("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 0);

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}

Example 9

Source File: TestBloomFilters.java From hadoop with Apache License 2.0

5 votes

@Test
public void testNot() {
  BloomFilter bf = new BloomFilter(8, 1, Hash.JENKINS_HASH);
  bf.bits = BitSet.valueOf(new byte[] { (byte) 0x95 });
  BitSet origBitSet = (BitSet) bf.bits.clone();
  bf.not();
  assertFalse("BloomFilter#not should have inverted all bits",
              bf.bits.intersects(origBitSet));
}

Example 10

Source File: Filter.java From big-c with Apache License 2.0

5 votes

@Override
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}

Example 11

Source File: TestBloomFilters.java From big-c with Apache License 2.0

5 votes

@Test
public void testCountingBloomFilter() {
  int hashId = Hash.JENKINS_HASH;

  CountingBloomFilter filter = new CountingBloomFilter(bitSize,
      hashFunctionNumber, hashId);

  Key key = new Key(new byte[] { 48, 48 });

  filter.add(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 1);

  filter.add(key);
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 2);

  filter.delete(key);
  assertTrue("CountingBloomFilter.membership error ",
      filter.membershipTest(key));

  filter.delete(key);
  assertFalse("CountingBloomFilter.membership error ",
      filter.membershipTest(key));
  assertTrue("CountingBloomFilter.approximateCount error",
      filter.approximateCount(key) == 0);

  BloomFilterCommonTester.of(hashId, numInsertions)
      .withFilterInstance(filter)
      .withTestCases(ImmutableSet.of(BloomFilterTestStrategy.KEY_TEST_STRATEGY,
              BloomFilterTestStrategy.ADD_KEYS_STRATEGY,
              BloomFilterTestStrategy.EXCEPTIONS_CHECK_STRATEGY,
              BloomFilterTestStrategy.ODD_EVEN_ABSENT_STRATEGY,
              BloomFilterTestStrategy.WRITE_READ_STRATEGY,
              BloomFilterTestStrategy.FILTER_OR_STRATEGY,
              BloomFilterTestStrategy.FILTER_XOR_STRATEGY)).test();
}

Example 12

Source File: TestBloomFilters.java From big-c with Apache License 2.0

5 votes

@Test
public void testNot() {
  BloomFilter bf = new BloomFilter(8, 1, Hash.JENKINS_HASH);
  bf.bits = BitSet.valueOf(new byte[] { (byte) 0x95 });
  BitSet origBitSet = (BitSet) bf.bits.clone();
  bf.not();
  assertFalse("BloomFilter#not should have inverted all bits",
              bf.bits.intersects(origBitSet));
}

Example 13

Source File: InternalFilter.java From hudi with Apache License 2.0

5 votes

@Override
public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}

Example 14

Source File: BuildBloomBase.java From spork with Apache License 2.0

5 votes

private int convertHashType(String hashType) {
    if (hashType.toLowerCase().contains("jenkins")) {
        return Hash.JENKINS_HASH;
    } else if (hashType.toLowerCase().contains("murmur")) {
        return Hash.MURMUR_HASH;
    } else {
        throw new RuntimeException("Unknown hash type " + hashType +
            ".  Valid values are jenkins and murmur.");
    }
}

Example 15

Source File: Filter.java From RDFS with Apache License 2.0

5 votes

public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}

Example 16

Source File: Filter.java From hadoop-gpu with Apache License 2.0

5 votes

public void readFields(DataInput in) throws IOException {
  int ver = in.readInt();
  if (ver > 0) { // old unversioned format
    this.nbHash = ver;
    this.hashType = Hash.JENKINS_HASH;
  } else if (ver == VERSION) {
    this.nbHash = in.readInt();
    this.hashType = in.readByte();
  } else {
    throw new IOException("Unsupported version: " + ver);
  }
  this.vectorSize = in.readInt();
  this.hash = new HashFunction(this.vectorSize, this.nbHash, this.hashType);
}