org.apache.rocketmq.common.message.MessageQueue Java Examples

public Map<MessageQueue, Long> resetOffsetByTimestamp(String topic, String group, long timestamp, boolean isForce,
    boolean isC)
    throws RemotingException, MQBrokerException, InterruptedException, MQClientException {
    TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic);
    List<BrokerData> brokerDatas = topicRouteData.getBrokerDatas();
    Map<MessageQueue, Long> allOffsetTable = new HashMap<MessageQueue, Long>();
    if (brokerDatas != null) {
        for (BrokerData brokerData : brokerDatas) {
            String addr = brokerData.selectBrokerAddr();
            if (addr != null) {
                Map<MessageQueue, Long> offsetTable =
                    this.mqClientInstance.getMQClientAPIImpl().invokeBrokerToResetOffset(addr, topic, group, timestamp, isForce,
                        timeoutMillis, isC);
                if (offsetTable != null) {
                    allOffsetTable.putAll(offsetTable);
                }
            }
        }
    }
    return allOffsetTable;
}
 

private void truncateMessageQueueNotMyTopic() {
//        获取订阅信息
        Map<String, SubscriptionData> subTable = this.getSubscriptionInner();

//        遍历处理队列
        for (MessageQueue mq : this.processQueueTable.keySet()) {
            if (!subTable.containsKey(mq.getTopic())) {

//                删除处理队列
                ProcessQueue pq = this.processQueueTable.remove(mq);
                if (pq != null) {
                    pq.setDropped(true);
                    log.info("doRebalance, {}, truncateMessageQueueNotMyTopic remove unnecessary mq, {}", consumerGroup, mq);
                }
            }
        }
    }
 

@Override
public void updateOffset(MessageQueue mq, long offset, boolean increaseOnly) {
    if (mq != null) {
        AtomicLong offsetOld = this.offsetTable.get(mq);
        if (null == offsetOld) {
            offsetOld = this.offsetTable.putIfAbsent(mq, new AtomicLong(offset));
        }

        if (null != offsetOld) {
            if (increaseOnly) {
                MixAll.compareAndIncreaseOnly(offsetOld, offset);
            } else {
                offsetOld.set(offset);
            }
        }
    }
}
 

@BeforeClass
public static void init() throws MQClientException, RemotingException, InterruptedException, MQBrokerException, NoSuchFieldException, IllegalAccessException {

    DefaultMQProducer defaultMQProducer = mock(DefaultMQProducer.class);
    SendResult sendResult = new SendResult();
    sendResult.setMessageQueue(new MessageQueue());
    sendResult.getMessageQueue().setBrokerName("broker1");
    sendResult.getMessageQueue().setQueueId(1);
    sendResult.setSendStatus(SendStatus.SEND_OK);
    sendResult.setMsgId("fgwejigherughwueyutyu4t4343t43");

    when(defaultMQProducer.send(any(Message.class))).thenReturn(sendResult);
    when(defaultMQProducer.send(any(Message.class), any(MessageQueue.class))).thenReturn(sendResult);

    Field producerField = SendMessageCommand.class.getDeclaredField("producer");
    producerField.setAccessible(true);
    producerField.set(sendMessageCommand, defaultMQProducer);
}
 

public void send(CarreraRequest request, MessageQueueSelector messageQueueSelector)
    throws RemotingException, MQClientException, InterruptedException, MQBrokerException {
    TopicPublishInfo topicInfo = clusterProducer.getRocketMQProducerByIndex(0).getDefaultMQProducerImpl().getTopicPublishInfoTable().get(request.getTopic());
    if (topicInfo == null || !topicInfo.ok()) { //new topic
        sendSingleMessage(request);
        return;
    }

    MessageQueue mq = messageQueueSelector.select(topicInfo.getMessageQueueList(), null, request);
    request.setMessageQueue(mq);

    requestQMap.computeIfAbsent(mq.getBrokerName(), _name -> {
        Deque<CarreraRequest> q = new ConcurrentLinkedDeque<>();
        addRequestQueue(q, _name, config.getMaxEncodeWorkerForEachBroker());
        return q;
    }).add(request);
}
 

public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException {
    try {
        TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis);
        if (topicRouteData != null) {
            Set<MessageQueue> mqList = MQClientInstance.topicRouteData2TopicSubscribeInfo(topic, topicRouteData);
            if (!mqList.isEmpty()) {
                return mqList;
            } else {
                throw new MQClientException("Can not find Message Queue for this topic, " + topic + " Namesrv return empty", null);
            }
        }
    } catch (Exception e) {
        throw new MQClientException(
            "Can not find Message Queue for this topic, " + topic + FAQUrl.suggestTodo(FAQUrl.MQLIST_NOT_EXIST), //
            e);
    }

    throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null);
}
 

public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException {
    String brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
    if (null == brokerAddr) {
        this.mQClientFactory.updateTopicRouteInfoFromNameServer(mq.getTopic());
        brokerAddr = this.mQClientFactory.findBrokerAddressInPublish(mq.getBrokerName());
    }

    if (brokerAddr != null) {
        try {
            return this.mQClientFactory.getMQClientAPIImpl().searchOffset(brokerAddr, mq.getTopic(), mq.getQueueId(), timestamp,
                timeoutMillis);
        } catch (Exception e) {
            throw new MQClientException("Invoke Broker[" + brokerAddr + "] exception", e);
        }
    }

    throw new MQClientException("The broker[" + mq.getBrokerName() + "] not exist", null);
}
 

@Override
public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {
    MessageQueueListener messageQueueListener = this.litePullConsumerImpl.getDefaultLitePullConsumer().getMessageQueueListener();
    if (messageQueueListener != null) {
        try {
            messageQueueListener.messageQueueChanged(topic, mqAll, mqDivided);
        } catch (Throwable e) {
            log.error("messageQueueChanged exception", e);
        }
    }
}
 

@Test
public void testTwoConsumerSubTag() {
    int msgSize = 10;

    RMQBroadCastConsumer consumer1 = getBroadCastConsumer(nsAddr, topic, "*",
        new RMQOrderListener());
    RMQBroadCastConsumer consumer2 = getBroadCastConsumer(nsAddr,
        consumer1.getConsumerGroup(), topic, "*", new RMQOrderListener());
    TestUtils.waitForSeconds(waitTime);

    List<MessageQueue> mqs = producer.getMessageQueue();
    MessageQueueMsg mqMsgs = new MessageQueueMsg(mqs, msgSize);
    producer.send(mqMsgs.getMsgsWithMQ());

    consumer1.getListner().waitForMessageConsume(producer.getAllMsgBody(), consumeTime);
    consumer2.getListner().waitForMessageConsume(producer.getAllMsgBody(), consumeTime);

    assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer1.getListner()).getMsgs()))
        .isEqualTo(true);
    assertThat(VerifyUtils.verifyOrder(((RMQOrderListener) consumer2.getListner()).getMsgs()))
        .isEqualTo(true);
}
 

public synchronized void registerTopicMessageQueueChangeListener(String topic,
    TopicMessageQueueChangeListener listener) throws MQClientException {
    if (topic == null || listener == null) {
        throw new MQClientException("Topic or listener is null", null);
    }
    if (topicMessageQueueChangeListenerMap.containsKey(topic)) {
        log.warn("Topic {} had been registered, new listener will overwrite the old one", topic);
    }
    topicMessageQueueChangeListenerMap.put(topic, listener);
    if (this.serviceState == ServiceState.RUNNING) {
        Set<MessageQueue> messageQueues = fetchMessageQueues(topic);
        messageQueuesForTopic.put(topic, messageQueues);
    }
}
 

@Override
public void submitConsumeRequest(
    final List<MessageExt> msgs,
    final ProcessQueue processQueue,
    final MessageQueue messageQueue,
    final boolean dispathToConsume) {
    if (!this.stopped && dispathToConsume) {
        ConsumeRequest consumeRequest = new ConsumeRequest(processQueue, messageQueue);
        this.consumeExecutor.submit(consumeRequest);
    }
}
 

public Map<MessageQueue, Long> invokeBrokerToResetOffset(final String addr, final String topic, final String group,
    final long timestamp, final boolean isForce, final long timeoutMillis, boolean isC)
    throws RemotingException, MQClientException, InterruptedException {
    ResetOffsetRequestHeader requestHeader = new ResetOffsetRequestHeader();
    requestHeader.setTopic(topic);
    requestHeader.setGroup(group);
    requestHeader.setTimestamp(timestamp);
    requestHeader.setForce(isForce);

    RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_RESET_OFFSET, requestHeader);
    if (isC) {
        request.setLanguage(LanguageCode.CPP);
    }

    RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
        request, timeoutMillis);
    assert response != null;
    switch (response.getCode()) {
        case ResponseCode.SUCCESS: {
            if (response.getBody() != null) {
                ResetOffsetBody body = ResetOffsetBody.decode(response.getBody(), ResetOffsetBody.class);
                return body.getOffsetTable();
            }
        }
        default:
            break;
    }

    throw new MQClientException(response.getCode(), response.getRemark());
}
 

/**
 * 添加任务
 * @param topic topic
 * @param mqNewSet 最新被分配的队列
 */
public void putTask(String topic, Set<MessageQueue> mqNewSet) {
    //一个队列一个线程
    Iterator<Entry<MessageQueue, PullTaskImpl>> it = this.taskTable.entrySet().iterator();
    while (it.hasNext()) {
        Entry<MessageQueue, PullTaskImpl> next = it.next();
        //找到指定topic的
        if (next.getKey().getTopic().equals(topic)) {
            //如果mqNewSet不包含此queue。就移除
            if (!mqNewSet.contains(next.getKey())) {
                next.getValue().setCancelled(true);
                it.remove();
            }
        }
    }

    /*
     * 添加新的被分配到的MessageQueue
     */
    for (MessageQueue mq : mqNewSet) {
        if (!this.taskTable.containsKey(mq)) {
            PullTaskImpl command = new PullTaskImpl(mq);
            this.taskTable.put(mq, command);
            this.scheduledThreadPoolExecutor.schedule(command, 0, TimeUnit.MILLISECONDS);

        }
    }
}
 

@Test
public void testCurrentCIDNotExists() {
    String currentCID = String.valueOf(Integer.MAX_VALUE);
    List<String> consumerIdList = createConsumerIdList(2);
    List<MessageQueue> messageQueueList = createMessageQueueList(6);
    List<MessageQueue> result = new AllocateMessageQueueConsistentHash().allocate("", currentCID, messageQueueList, consumerIdList);
    printMessageQueue(result, "testCurrentCIDNotExists");
    Assert.assertEquals(result.size(), 0);
}
 

public synchronized boolean lockOneMQ(final MessageQueue mq) {
    if (!this.stopped) {
        return this.defaultMQPushConsumerImpl.getRebalanceImpl().lock(mq);
    }

    return false;
}
 

public Object fetchLockObject(final MessageQueue mq) {
    Object objLock = this.mqLockTable.get(mq);
    if (null == objLock) {
        objLock = new Object();
        Object prevLock = this.mqLockTable.putIfAbsent(mq, objLock);
        if (prevLock != null) {
            objLock = prevLock;
        }
    }

    return objLock;
}
 

public boolean putOpMessage(MessageExt messageExt, String opType) {
        MessageQueue messageQueue = new MessageQueue(messageExt.getTopic(),
            this.brokerController.getBrokerConfig().getBrokerName(), messageExt.getQueueId());
        if (TransactionalMessageUtil.REMOVETAG.equals(opType)) {
//            事务消息提交或回滚时添加删除标记=》
            return addRemoveTagInTransactionOp(messageExt, messageQueue);
        }
        return true;
    }
 

public void printMessageQueue(List<MessageQueue> messageQueueList, String name) {
    if (messageQueueList == null || messageQueueList.size() < 1)
        return;
    System.out.println(name + ".......................................start");
    for (MessageQueue messageQueue : messageQueueList) {
        System.out.println(messageQueue);
    }
    System.out.println(name + ".......................................end");
}
 

public void testWhenConsumerIDCIsLess(int brokerIDCSize, int consumerIDCLess, int queueSize, int consumerSize, boolean print) {
    if (print) {
        System.out.println("Test : IDCSize = "+ brokerIDCSize +" queueSize = " + queueSize +" consumerSize = " + consumerSize);
    }
    Set<String> healthyIDC = new TreeSet<String>();
    List<String> cidAll = prepareConsumer(brokerIDCSize - consumerIDCLess, consumerSize);
    List<MessageQueue> mqAll = prepareMQ(brokerIDCSize, queueSize);
    for (String cid : cidAll) {
        healthyIDC.add(machineRoomResolver.consumerDeployIn(cid));
    }

    List<MessageQueue> resAll = new ArrayList<MessageQueue>();
    Map<String, List<MessageQueue>> idc2Res = new TreeMap<String, List<MessageQueue>>();
    for (String currentID : cidAll) {
        String currentIDC = machineRoomResolver.consumerDeployIn(currentID);
        List<MessageQueue> res = allocateMessageQueueStrategy.allocate("Test-C-G",currentID,mqAll,cidAll);
        if (print) {
            System.out.println("cid: "+currentID+"--> res :" +res);
        }
        if ( !idc2Res.containsKey(currentIDC)) {
            idc2Res.put(currentIDC, new ArrayList<MessageQueue>());
        }
        idc2Res.get(currentIDC).addAll(res);
        resAll.addAll(res);
    }

    for (String consumerIDC : healthyIDC) {
        List<MessageQueue> resInOneIDC = idc2Res.get(consumerIDC);
        List<MessageQueue> mqInThisIDC = createMessageQueueList(consumerIDC,queueSize);
        Assert.assertTrue(resInOneIDC.containsAll(mqInThisIDC));
    }

    Assert.assertTrue(hasAllocateAllQ(cidAll,mqAll,resAll));
    if (print) {
        System.out.println("-------------------------------------------------------------------");
    }
}
 

public List<MessageQueue> fetchPublishMessageQueues(String topic) throws MQClientException {
    try {
        TopicRouteData topicRouteData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, timeoutMillis);
        if (topicRouteData != null) {
            TopicPublishInfo topicPublishInfo = MQClientInstance.topicRouteData2TopicPublishInfo(topic, topicRouteData);
            if (topicPublishInfo != null && topicPublishInfo.ok()) {
                return topicPublishInfo.getMessageQueueList();
            }
        }
    } catch (Exception e) {
        throw new MQClientException("Can not find Message Queue for this topic, " + topic, e);
    }

    throw new MQClientException("Unknow why, Can not find Message Queue for this topic, " + topic, null);
}
 

public void sendOneWay(Object msg, MessageQueue mq) {
    Message metaqMsg = (Message) msg;
    try {
        producer.sendOneway(metaqMsg, mq);
        msgBodys.addData(new String(metaqMsg.getBody()));
        originMsgs.addData(msg);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 

public Map<String, Map<MessageQueue, Long>> invokeBrokerToGetConsumerStatus(final String addr, final String topic,
    final String group,
    final String clientAddr,
    final long timeoutMillis) throws RemotingException, MQClientException, InterruptedException {
    GetConsumerStatusRequestHeader requestHeader = new GetConsumerStatusRequestHeader();
    requestHeader.setTopic(topic);
    requestHeader.setGroup(group);
    requestHeader.setClientAddr(clientAddr);

    RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.INVOKE_BROKER_TO_GET_CONSUMER_STATUS, requestHeader);

    RemotingCommand response = this.remotingClient.invokeSync(MixAll.brokerVIPChannel(this.clientConfig.isVipChannelEnabled(), addr),
        request, timeoutMillis);
    assert response != null;
    switch (response.getCode()) {
        case ResponseCode.SUCCESS: {
            if (response.getBody() != null) {
                GetConsumerStatusBody body = GetConsumerStatusBody.decode(response.getBody(), GetConsumerStatusBody.class);
                return body.getConsumerTable();
            }
        }
        default:
            break;
    }

    throw new MQClientException(response.getCode(), response.getRemark());
}
 

private boolean verifyAllocateAll(List<String> cidAll, List<MessageQueue> mqAll,
    List<MessageQueue> allocatedResAll) {
    if (cidAll.isEmpty()) {
        return allocatedResAll.isEmpty();
    }
    return mqAll.containsAll(allocatedResAll) && allocatedResAll.containsAll(mqAll);
}
 

@Test(expected = java.lang.NullPointerException.class)
public void testSendMQNull() throws Exception {
    Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());
    DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr);
    MessageQueue messageQueue = null;
    producer.send(msg, messageQueue, SendCallBackFactory.getSendCallBack());
}
 

@Override
public Map<String, Map<MessageQueue, Long>> getConsumeStatus(String topic, String group,
    String clientAddr) throws RemotingException,
    MQBrokerException, InterruptedException, MQClientException {
    TopicRouteData topicRouteData = this.examineTopicRouteInfo(topic);
    List<BrokerData> brokerDatas = topicRouteData.getBrokerDatas();
    if (brokerDatas != null && brokerDatas.size() > 0) {
        String addr = brokerDatas.get(0).selectBrokerAddr();
        if (addr != null) {
            return this.mqClientInstance.getMQClientAPIImpl().invokeBrokerToGetConsumerStatus(addr, topic, group, clientAddr,
                timeoutMillis);
        }
    }
    return Collections.EMPTY_MAP;
}
 

@Test(expected = java.lang.NullPointerException.class)
public void testSendMQNull() throws Exception {
    Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());
    DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr);
    MessageQueue messageQueue = null;
    producer.sendOneway(msg, messageQueue);
}
 

@Override
public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) {
    String srcGroup = commandLine.getOptionValue("s").trim();
    String destGroup = commandLine.getOptionValue("d").trim();
    String topic = commandLine.getOptionValue("t").trim();

    DefaultMQAdminExt defaultMQAdminExt = new DefaultMQAdminExt(rpcHook);
    defaultMQAdminExt.setInstanceName("admin-" + Long.toString(System.currentTimeMillis()));

    try {
        defaultMQAdminExt.start();
        ConsumeStats consumeStats = defaultMQAdminExt.examineConsumeStats(srcGroup);
        Set<MessageQueue> mqs = consumeStats.getOffsetTable().keySet();
        if (!mqs.isEmpty()) {
            TopicRouteData topicRoute = defaultMQAdminExt.examineTopicRouteInfo(topic);
            for (MessageQueue mq : mqs) {
                String addr = null;
                for (BrokerData brokerData : topicRoute.getBrokerDatas()) {
                    if (brokerData.getBrokerName().equals(mq.getBrokerName())) {
                        addr = brokerData.selectBrokerAddr();
                        break;
                    }
                }
                long offset = consumeStats.getOffsetTable().get(mq).getBrokerOffset();
                if (offset >= 0) {
                    defaultMQAdminExt.updateConsumeOffset(addr, destGroup, mq, offset);
                }
            }
        }
        System.out.printf("clone group offset success. srcGroup[%s], destGroup=[%s], topic[%s]",
            srcGroup, destGroup, topic);
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        defaultMQAdminExt.shutdown();
    }
}
 

@Test(expected = java.lang.NullPointerException.class)
public void testSendMQNull() throws Exception {
    Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());
    DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr);
    MessageQueue messageQueue = null;
    producer.sendOneway(msg, messageQueue);
}
 

public void removeAssignedMessageQueue(String topic) {
    synchronized (this.assignedMessageQueueState) {
        Iterator<Map.Entry<MessageQueue, MessageQueueState>> it = this.assignedMessageQueueState.entrySet().iterator();
        while (it.hasNext()) {
            Map.Entry<MessageQueue, MessageQueueState> next = it.next();
            if (next.getKey().getTopic().equals(topic)) {
                it.remove();
            }
        }
    }
}
 

@Test(expected = java.lang.NullPointerException.class)
public void testSendMQNull() throws Exception {
    Message msg = new Message(topic, RandomUtils.getStringByUUID().getBytes());
    DefaultMQProducer producer = ProducerFactory.getRMQProducer(nsAddr);
    MessageQueue messageQueue = null;
    producer.send(msg, messageQueue, SendCallBackFactory.getSendCallBack());
}