Java Code Examples for org.apache.rocketmq.filter.util.BitsArray#create()

The following examples show how to use org.apache.rocketmq.filter.util.BitsArray#create() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: BloomFilterTest.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

6 votes

@Test
public void testCheckFalseHit() {
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int falseHit = 0;
    for (int i = 0; i < bloomFilter.getN(); i++) {
        String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
        int[] bitPos = bloomFilter.calcBitPositions(str);

        if (bloomFilter.checkFalseHit(bitPos, bits)) {
            falseHit++;
        }

        bloomFilter.hashTo(bitPos, bits);
    }

    assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
}

Example 2

Source File: BloomFilterTest.java From rocketmq-4.3.0 with Apache License 2.0

6 votes

@Test
public void testCheckFalseHit() {
    BloomFilter bloomFilter = BloomFilter.createByFn(1, 300);
    BitsArray bits = BitsArray.create(bloomFilter.getM());
    int falseHit = 0;
    for (int i = 0; i < bloomFilter.getN(); i++) {
        String str = randomString((new Random(System.nanoTime())).nextInt(127) + 10);
        int[] bitPos = bloomFilter.calcBitPositions(str);

        if (bloomFilter.checkFalseHit(bitPos, bits)) {
            falseHit++;
        }

        bloomFilter.hashTo(bitPos, bits);
    }

    assertThat(falseHit).isLessThanOrEqualTo(bloomFilter.getF() * bloomFilter.getN() / 100);
}

Example 3

Source File: BitsArrayTest.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

5 votes

@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[]{(byte) 0xff, 0x00});
    BitsArray b2 = BitsArray.create(new byte[]{0x00, (byte) 0xff});

    b1.or(b2);

    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}

Example 4

Source File: BitsArrayTest.java From DDMQ with Apache License 2.0

5 votes

@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[] {(byte) 0xff, 0x00});
    BitsArray b2 = BitsArray.create(new byte[] {0x00, (byte) 0xff});

    b1.or(b2);

    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}

Example 5

Source File: BitsArrayTest.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

5 votes

BitsArray gen(int bitCount) {
    BitsArray bitsArray = BitsArray.create(bitCount);

    for (int i = 0; i < bitCount / Byte.SIZE; i++) {
        bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
        }
    }

    return bitsArray;
}

Example 6

Source File: BitsArrayTest.java From rocketmq-4.3.0 with Apache License 2.0

5 votes

@Test
public void testOr() {
    BitsArray b1 = BitsArray.create(new byte[] {(byte) 0xff, 0x00});
    BitsArray b2 = BitsArray.create(new byte[] {0x00, (byte) 0xff});

    b1.or(b2);

    for (int i = 0; i < b1.bitLength(); i++) {
        assertThat(b1.getBit(i)).isTrue();
    }
}

Example 7

Source File: BitsArrayTest.java From rocketmq with Apache License 2.0

5 votes

BitsArray gen(int bitCount) {
    BitsArray bitsArray = BitsArray.create(bitCount);

    for (int i = 0; i < bitCount / Byte.SIZE; i++) {
        bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
        }
    }

    return bitsArray;
}

Example 8

Source File: BitsArrayTest.java From DDMQ with Apache License 2.0

5 votes

BitsArray gen(int bitCount) {
    BitsArray bitsArray = BitsArray.create(bitCount);

    for (int i = 0; i < bitCount / Byte.SIZE; i++) {
        bitsArray.setByte(i, (byte) (new Random(System.currentTimeMillis())).nextInt(0xff));
        try {
            Thread.sleep(2);
        } catch (InterruptedException e) {
        }
    }

    return bitsArray;
}

Example 9

Source File: BloomFilterTest.java From rocketmq-4.3.0 with Apache License 2.0

4 votes

@Test
public void testHashTo() {
    String cid = "CID_abc_efg";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    int[] bitPos = bloomFilter.calcBitPositions(cid);

    bloomFilter.hashTo(cid, bits);

    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}

Example 10

Source File: BloomFilterTest.java From rocketmq with Apache License 2.0

4 votes

@Test
public void testIsHit() {
    String cid = "CID_abc_efg";
    String cid2 = "CID_abc_123";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    bloomFilter.hashTo(cid, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();

    bloomFilter.hashTo(cid2, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(bloomFilter.isHit(cid2, bits)).isTrue();
}

Example 11

Source File: BloomFilterTest.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

4 votes

@Test
public void testHashTo() {
    String cid = "CID_abc_efg";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    int[] bitPos = bloomFilter.calcBitPositions(cid);

    bloomFilter.hashTo(cid, bits);

    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}

Example 12

Source File: ExpressionMessageFilter.java From DDMQ with Apache License 2.0

4 votes

@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }

    if (subscriptionData.isClassFilterMode()) {
        return true;
    }

    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {

        if (tagsCode == null) {
            return true;
        }

        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }

        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null
            || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }

        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }

        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid
            || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }

        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData
                + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }

    return true;
}

Example 13

Source File: ExpressionMessageFilter.java From rocketmq-4.3.0 with Apache License 2.0

4 votes

@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }

    if (subscriptionData.isClassFilterMode()) {
        return true;
    }

    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {

        if (tagsCode == null) {
            return true;
        }

        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }

        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null
            || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }

        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }

        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid
            || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }

        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData
                + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }

    return true;
}

Example 14

Source File: ExpressionMessageFilter.java From DDMQ with Apache License 2.0

4 votes

@Override
public boolean isMatchedByConsumeQueue(Long tagsCode, ConsumeQueueExt.CqExtUnit cqExtUnit) {
    if (null == subscriptionData) {
        return true;
    }

    if (subscriptionData.isClassFilterMode()) {
        return true;
    }

    // by tags code.
    if (ExpressionType.isTagType(subscriptionData.getExpressionType())) {

        if (tagsCode == null) {
            return true;
        }

        if (subscriptionData.getSubString().equals(SubscriptionData.SUB_ALL)) {
            return true;
        }

        return subscriptionData.getCodeSet().contains(tagsCode.intValue());
    } else {
        // no expression or no bloom
        if (consumerFilterData == null || consumerFilterData.getExpression() == null
            || consumerFilterData.getCompiledExpression() == null || consumerFilterData.getBloomFilterData() == null) {
            return true;
        }

        // message is before consumer
        if (cqExtUnit == null || !consumerFilterData.isMsgInLive(cqExtUnit.getMsgStoreTime())) {
            log.debug("Pull matched because not in live: {}, {}", consumerFilterData, cqExtUnit);
            return true;
        }

        byte[] filterBitMap = cqExtUnit.getFilterBitMap();
        BloomFilter bloomFilter = this.consumerFilterManager.getBloomFilter();
        if (filterBitMap == null || !this.bloomDataValid
            || filterBitMap.length * Byte.SIZE != consumerFilterData.getBloomFilterData().getBitNum()) {
            return true;
        }

        BitsArray bitsArray = null;
        try {
            bitsArray = BitsArray.create(filterBitMap);
            boolean ret = bloomFilter.isHit(consumerFilterData.getBloomFilterData(), bitsArray);
            log.debug("Pull {} by bit map:{}, {}, {}", ret, consumerFilterData, bitsArray, cqExtUnit);
            return ret;
        } catch (Throwable e) {
            log.error("bloom filter error, sub=" + subscriptionData
                + ", filter=" + consumerFilterData + ", bitMap=" + bitsArray, e);
        }
    }

    return true;
}

Example 15

Source File: CommitLogDispatcherCalcBitMapTest.java From DDMQ with Apache License 2.0

4 votes

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}

Example 16

Source File: CommitLogDispatcherCalcBitMapTest.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

4 votes

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);


    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}

Example 17

Source File: BloomFilterTest.java From rocketmq with Apache License 2.0

4 votes

@Test
public void testHashTo() {
    String cid = "CID_abc_efg";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    int[] bitPos = bloomFilter.calcBitPositions(cid);

    bloomFilter.hashTo(cid, bits);

    for (int bit : bitPos) {
        assertThat(bits.getBit(bit)).isTrue();
    }
}

Example 18

Source File: BloomFilterTest.java From DDMQ with Apache License 2.0

4 votes

@Test
public void testIsHit() {
    String cid = "CID_abc_efg";
    String cid2 = "CID_abc_123";

    BloomFilter bloomFilter = BloomFilter.createByFn(10, 20);

    BitsArray bits = BitsArray.create(bloomFilter.getM());

    bloomFilter.hashTo(cid, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(!bloomFilter.isHit(cid2, bits)).isTrue();

    bloomFilter.hashTo(cid2, bits);

    assertThat(bloomFilter.isHit(cid, bits)).isTrue();
    assertThat(bloomFilter.isHit(cid2, bits)).isTrue();
}

Example 19

Source File: BitsArrayTest.java From DDMQ with Apache License 2.0

3 votes

@Test
public void testConstructor() {
    BitsArray bitsArray = BitsArray.create(8);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();

    bitsArray = BitsArray.create(9);

    assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();

    bitsArray = BitsArray.create(7);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();
}

Example 20

Source File: BitsArrayTest.java From rocketmq with Apache License 2.0

3 votes

@Test
public void testConstructor() {
    BitsArray bitsArray = BitsArray.create(8);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 8).isTrue();

    bitsArray = BitsArray.create(9);

    assertThat(bitsArray.byteLength() == 2 && bitsArray.bitLength() == 9).isTrue();

    bitsArray = BitsArray.create(7);

    assertThat(bitsArray.byteLength() == 1 && bitsArray.bitLength() == 7).isTrue();
}