org.apache.rocketmq.store.DispatchRequest Java Examples

The following examples show how to use org.apache.rocketmq.store.DispatchRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: IndexService.java From DDMQ with Apache License 2.0

5 votes

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #2

Source File: IndexService.java From rocketmq with Apache License 2.0

5 votes

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #3

Source File: DLedgerCommitLog.java From rocketmq with Apache License 2.0

5 votes

@Override
public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC,
    final boolean readBody) {
    if (isInrecoveringOldCommitlog) {
        return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
    }
    try {
        int bodyOffset = DLedgerEntry.BODY_OFFSET;
        int pos = byteBuffer.position();
        int magic = byteBuffer.getInt();
        //In dledger, this field is size, it must be gt 0, so it could prevent collision
        int magicOld = byteBuffer.getInt();
        if (magicOld == CommitLog.BLANK_MAGIC_CODE || magicOld == CommitLog.MESSAGE_MAGIC_CODE) {
            byteBuffer.position(pos);
            return super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
        }
        if (magic == MmapFileList.BLANK_MAGIC_CODE) {
            return new DispatchRequest(0, true);
        }
        byteBuffer.position(pos + bodyOffset);
        DispatchRequest dispatchRequest = super.checkMessageAndReturnSize(byteBuffer, checkCRC, readBody);
        if (dispatchRequest.isSuccess()) {
            dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);
        } else if (dispatchRequest.getMsgSize() > 0) {
            dispatchRequest.setBufferSize(dispatchRequest.getMsgSize() + bodyOffset);
        }
        return dispatchRequest;
    } catch (Throwable ignored) {
    }

    return new DispatchRequest(-1, false /* success */);
}

Example #4

Source File: MessageStoreWithFilterTest.java From rocketmq with Apache License 2.0

5 votes

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}

Example #5

Source File: CommitLogDispatcherCalcBitMapTest.java From rocketmq with Apache License 2.0

5 votes

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}

Example #6

Source File: IndexService.java From rocketmq_trans_message with Apache License 2.0

5 votes

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #7

Source File: TransactionRecordFlush2DBService.java From rocketmq_trans_message with Apache License 2.0

5 votes

public void appendPreparedTransaction(DispatchRequest dispatchRequest) {
    try {
        flowController.acquire(1);
        while (true) {
            DispatchRequestCollections requests = dispatchRequestBufferQueue.peek();
            if (requests.latch.getAndIncrement() >= 0) {
                requests.requestlist.add(dispatchRequest);
                break;
            }
        }
    } catch (InterruptedException e) {
        log.error("putDispatchRequest interrupted");
    }
}

Example #8

Source File: TransactionStateDBService.java From rocketmq_trans_message with Apache License 2.0

5 votes

@Override
public boolean updateTransactionState(DispatchRequest request) {
    try {
        transactionRecordFlush2DBService.appendPreparedTransaction(request);
        return true;
    } catch (Throwable e) {
        log.error(e.getMessage(), e);
        return false;
    }
}

Example #9

Source File: TransactionStateDBService.java From rocketmq_trans_message with Apache License 2.0

5 votes

@Override
public boolean appendPreparedTransaction(DispatchRequest request) {
    try {
        transactionRecordFlush2DBService.appendPreparedTransaction(request);
        return true;
    } catch (Throwable e) {
        log.error(e.getMessage(), e);
        return false;
    }
}

Example #10

Source File: IndexService.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

5 votes

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();//如果indexFile有就获取没有就创建
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());//写入索引消息
    }

    return indexFile;
}

Example #11

Source File: CommitLogDispatcherCalcBitMapTest.java From rocketmq-all-4.1.0-incubating with Apache License 2.0

5 votes

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}

Example #12

Source File: MessageStoreWithFilterTest.java From DDMQ with Apache License 2.0

5 votes

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}

Example #13

Source File: CommitLogDispatcherCalcBitMapTest.java From DDMQ with Apache License 2.0

5 votes

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}

Example #14

Source File: IndexService.java From rocketmq with Apache License 2.0

5 votes

/**
 * 只有在消息中指定了keys或者uniqueKey,才会构建索引
 *
 * @param indexFile
 * @param msg
 * @param idxKey    topic#keys;topic#uniqueKey
 * @return
 */
private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #15

Source File: MessageStoreWithFilterTest.java From rocketmq-4.3.0 with Apache License 2.0

5 votes

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}

Example #16

Source File: CommitLogDispatcherCalcBitMapTest.java From DDMQ with Apache License 2.0

5 votes

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}

Example #17

Source File: MessageStoreWithFilterTest.java From DDMQ with Apache License 2.0

5 votes

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}

Example #18

Source File: IndexService.java From DDMQ with Apache License 2.0

5 votes

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #19

Source File: CommitLogDispatcherCalcBitMapTest.java From rocketmq-4.3.0 with Apache License 2.0

5 votes

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}

Example #20

Source File: IndexService.java From rocketmq-read with Apache License 2.0

5 votes

/**
 * 添加消息
 * @param indexFile ;
 * @param msg ;
 * @param idxKey ;
 * @return ;
 */
private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #21

Source File: IndexService.java From rocketmq-4.3.0 with Apache License 2.0

5 votes

private IndexFile putKey(IndexFile indexFile, DispatchRequest msg, String idxKey) {
    for (boolean ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp()); !ok; ) {
        log.warn("Index file [" + indexFile.getFileName() + "] is full, trying to create another one");

        indexFile = retryGetAndCreateIndexFile();
        if (null == indexFile) {
            return null;
        }

        ok = indexFile.putKey(idxKey, msg.getCommitLogOffset(), msg.getStoreTimestamp());
    }

    return indexFile;
}

Example #22

Source File: CommitLogDispatcherCalcBitMapTest.java From rocketmq-read with Apache License 2.0

5 votes

@Test
public void testDispatch_blankFilterData() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 10; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNull();
    }
}

Example #23

Source File: MessageStoreWithFilterTest.java From rocketmq-read with Apache License 2.0

5 votes

protected DefaultMessageStore gen(ConsumerFilterManager filterManager) throws Exception {
    MessageStoreConfig messageStoreConfig = buildStoreConfig(
        commitLogFileSize, cqFileSize, true, cqExtFileSize
    );

    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);
    brokerConfig.setMaxErrorRateOfBloomFilter(20);
    brokerConfig.setExpectConsumerNumUseFilter(64);

    DefaultMessageStore master = new DefaultMessageStore(
        messageStoreConfig,
        new BrokerStatsManager(brokerConfig.getBrokerClusterName()),
        new MessageArrivingListener() {
            @Override
            public void arriving(String topic, int queueId, long logicOffset, long tagsCode,
                                 long msgStoreTime, byte[] filterBitMap, Map<String, String> properties) {
            }
        }
        , brokerConfig);

    master.getDispatcherList().addFirst(new CommitLogDispatcher() {
        @Override
        public void dispatch(DispatchRequest request) {
            try {
            } catch (Throwable e) {
                e.printStackTrace();
            }
        }
    });
    master.getDispatcherList().addFirst(new CommitLogDispatcherCalcBitMap(brokerConfig, filterManager));

    assertThat(master.load()).isTrue();

    master.start();

    return master;
}

Example #24

Source File: IndexService.java From rocketmq with Apache License 2.0

4 votes

/**
 * 只有在消息中指定了keys或者uniqueKey,才会构建索引
 * {@linkplain MessageSysFlag#TRANSACTION_ROLLBACK_TYPE}类型消息不会构建索引
 *
 * @param req
 */
public void buildIndex(DispatchRequest req) {
    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {
        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;
        String topic = msg.getTopic();
        String keys = msg.getKeys();  //默认值是""
        if (msg.getCommitLogOffset() < endPhyOffset) {  //待构建消息的偏移量应大于索引文件最大物理偏移量
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            // 事务类型为ROLLBACK_TYPE的消息不会进IndexFile
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }
        //若消息体中指定了uniqueKey属性(Producer创建时Client都会生成)
        if (req.getUniqKey() != null) {
            // key = topic#uniqueKey
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        if (keys != null && keys.length() > 0) {   //若消息中未指定keys及uniqueKey,则不会构建索引
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));  //key = topic#key
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                        return;
                    }
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}

Example #25

Source File: CommitLogDispatcherCalcBitMapTest.java From DDMQ with Apache License 2.0

4 votes

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}

Example #26

Source File: IndexService.java From rocketmq with Apache License 2.0

4 votes

public void buildIndex(DispatchRequest req) {
    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {
        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;
        String topic = msg.getTopic();
        String keys = msg.getKeys();
        if (msg.getCommitLogOffset() < endPhyOffset) {
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }

        if (req.getUniqKey() != null) {
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        if (keys != null && keys.length() > 0) {
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                        return;
                    }
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}

Example #27

Source File: IndexService.java From rocketmq-read with Apache License 2.0

4 votes

/**
 * 构造索引。当commitlog添加成功后构造索引
 * 构造keys的每一个key的索引文件和uniquekeys的文件
 * @param req req
 */
public void buildIndex(DispatchRequest req) {

    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {

        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;

        String topic = msg.getTopic();
        String keys = msg.getKeys();

        //已经提交过偏移量了，直接return
        if (msg.getCommitLogOffset() < endPhyOffset) {
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }

        if (req.getUniqKey() != null) {
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        //遍历keys循环添加索引文件
        if (keys != null && keys.length() > 0) {
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    //添加到索引文件
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                        return;
                    }
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}

Example #28

Source File: DLedgerCommitLog.java From rocketmq with Apache License 2.0

4 votes

@Override
public DispatchRequest checkMessageAndReturnSize(ByteBuffer byteBuffer, final boolean checkCRC) {
    return this.checkMessageAndReturnSize(byteBuffer, checkCRC, true);
}

Example #29

Source File: IndexService.java From DDMQ with Apache License 2.0

4 votes

public void buildIndex(DispatchRequest req) {
    IndexFile indexFile = retryGetAndCreateIndexFile();
    if (indexFile != null) {
        long endPhyOffset = indexFile.getEndPhyOffset();
        DispatchRequest msg = req;
        String topic = msg.getTopic();
        String keys = msg.getKeys();
        Map<String, String> properties = msg.getPropertiesMap();
        if (msg.getCommitLogOffset() < endPhyOffset) {
            return;
        }

        final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag());
        switch (tranType) {
            case MessageSysFlag.TRANSACTION_NOT_TYPE:
            case MessageSysFlag.TRANSACTION_PREPARED_TYPE:
            case MessageSysFlag.TRANSACTION_COMMIT_TYPE:
                break;
            case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE:
                return;
        }

        if (req.getUniqKey() != null) {
            indexFile = putKey(indexFile, msg, buildKey(topic, req.getUniqKey()));
            if (indexFile == null) {
                log.error("putKey error commitlog {} uniqkey {}", req.getCommitLogOffset(), req.getUniqKey());
                return;
            }
        }

        if (keys != null && keys.length() > 0) {
            String[] keyset = keys.split(MessageConst.KEY_SEPARATOR);
            for (int i = 0; i < keyset.length; i++) {
                String key = keyset[i];
                if (key.length() > 0) {
                    indexFile = putKey(indexFile, msg, buildKey(topic, key));
                    if (indexFile == null) {
                        log.error("putKey error commitlog {} key {}", req.getCommitLogOffset(), key);
                        return;
                    }
                }
            }
        }

        if (properties != null & properties.containsKey(MessageConst.PROPERTY_USER_TRACE_ID)) {
            String traceId = properties.get(MessageConst.PROPERTY_USER_TRACE_ID);
            if (traceId != null) {
                indexFile = putKey(indexFile, msg, buildKey(topic, traceId));
                if (indexFile == null) {
                    log.error("putKey error commitlog {} traceId {}", req.getCommitLogOffset(), traceId);
                    return;
                }
            }
        }
    } else {
        log.error("build index error, stop building index");
    }
}

Example #30

Source File: CommitLogDispatcherCalcBitMapTest.java From DDMQ with Apache License 2.0

4 votes

@Test
public void testDispatch_filterDataIllegal() {
    BrokerConfig brokerConfig = new BrokerConfig();
    brokerConfig.setEnableCalcFilterBitMap(true);

    ConsumerFilterManager filterManager = new ConsumerFilterManager();

    filterManager.register("topic0", "CID_0", "a is not null and a >= 5",
        ExpressionType.SQL92, System.currentTimeMillis());

    filterManager.register("topic0", "CID_1", "a is not null and a >= 15",
        ExpressionType.SQL92, System.currentTimeMillis());

    ConsumerFilterData nullExpression = filterManager.get("topic0", "CID_0");
    nullExpression.setExpression(null);
    nullExpression.setCompiledExpression(null);
    ConsumerFilterData nullBloomData = filterManager.get("topic0", "CID_1");
    nullBloomData.setBloomFilterData(null);

    CommitLogDispatcherCalcBitMap calcBitMap = new CommitLogDispatcherCalcBitMap(brokerConfig,
        filterManager);

    for (int i = 0; i < 1; i++) {
        Map<String, String> properties = new HashMap<String, String>(4);
        properties.put("a", String.valueOf(i * 10 + 5));

        String topic = "topic" + i;

        DispatchRequest dispatchRequest = new DispatchRequest(
            topic,
            0,
            i * 100 + 123,
            100,
            (long) ("tags" + i).hashCode(),
            System.currentTimeMillis(),
            i,
            null,
            UUID.randomUUID().toString(),
            0,
            0,
            properties
        );

        calcBitMap.dispatch(dispatchRequest);

        assertThat(dispatchRequest.getBitMap()).isNotNull();

        BitsArray bitsArray = BitsArray.create(dispatchRequest.getBitMap(),
            filterManager.getBloomFilter().getM());

        for (int j = 0; j < bitsArray.bitLength(); j++) {
            assertThat(bitsArray.getBit(j)).isFalse();
        }
    }
}