/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.rocketmq.client.impl.consumer; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.listener.MessageListener; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.hook.ConsumeMessageContext; import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.stat.ConsumerStatsManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.ConsumeStatus; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.ProcessQueueInfo; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; // public class DefaultMQPushConsumerImpl implements MQConsumerInner { /** * Delay some time when exception occur 当异常发生时,延迟一些时间 */ private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 3000; /** * Flow control interval */ private static final long PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL = 50; /** * Delay some time when suspend pull service 暂停拉拔服务时延迟一些时间 */ private static final long PULL_TIME_DELAY_MILLS_WHEN_SUSPEND = 1000; private static final long BROKER_SUSPEND_MAX_TIME_MILLIS = 1000 * 15; private static final long CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND = 1000 * 30; private final InternalLogger log = ClientLogger.getLog(); private final DefaultMQPushConsumer defaultMQPushConsumer; private final RebalanceImpl rebalanceImpl = new RebalancePushImpl(this); private final ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<FilterMessageHook>(); private final long consumerStartTimestamp = System.currentTimeMillis(); private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<ConsumeMessageHook>(); private final RPCHook rpcHook; private volatile ServiceState serviceState = ServiceState.CREATE_JUST; private MQClientInstance mQClientFactory; private PullAPIWrapper pullAPIWrapper; private volatile boolean pause = false; private boolean consumeOrderly = false; private MessageListener messageListenerInner; private OffsetStore offsetStore; private ConsumeMessageService consumeMessageService; private long queueFlowControlTimes = 0; private long queueMaxSpanFlowControlTimes = 0; public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) { this.defaultMQPushConsumer = defaultMQPushConsumer; this.rpcHook = rpcHook; } public void registerFilterMessageHook(final FilterMessageHook hook) { this.filterMessageHookList.add(hook); log.info("register FilterMessageHook Hook, {}", hook.hookName()); } // public boolean hasHook() { return !this.consumeMessageHookList.isEmpty(); } public void registerConsumeMessageHook(final ConsumeMessageHook hook) { this.consumeMessageHookList.add(hook); log.info("register consumeMessageHook Hook, {}", hook.hookName()); } // public void executeHookBefore(final ConsumeMessageContext context) { if (!this.consumeMessageHookList.isEmpty()) { for (ConsumeMessageHook hook : this.consumeMessageHookList) { try { hook.consumeMessageBefore(context); } catch (Throwable e) { } } } } // public void executeHookAfter(final ConsumeMessageContext context) { if (!this.consumeMessageHookList.isEmpty()) { for (ConsumeMessageHook hook : this.consumeMessageHookList) { try { hook.consumeMessageAfter(context); } catch (Throwable e) { } } } } public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { createTopic(key, newTopic, queueNum, 0); } // public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); } // public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException { Set<MessageQueue> result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); if (null == result) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); } if (null == result) { throw new MQClientException("The topic[" + topic + "] not exist", null); } return result; } public DefaultMQPushConsumer getDefaultMQPushConsumer() { return defaultMQPushConsumer; } public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); } public long maxOffset(MessageQueue mq) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } public long minOffset(MessageQueue mq) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().minOffset(mq); } public OffsetStore getOffsetStore() { return offsetStore; } public void setOffsetStore(OffsetStore offsetStore) { this.offsetStore = offsetStore; } // public void pullMessage(final PullRequest pullRequest) { // 从pull请求中获取处理队列 final ProcessQueue processQueue = pullRequest.getProcessQueue(); // 是否删除 if (processQueue.isDropped()) { log.info("the pull request[{}] is dropped.", pullRequest.toString()); return; } // 处理队列设置最后拉去的时间 pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis()); try { // 确认服务是否ok=》 this.makeSureStateOK(); } catch (MQClientException e) { log.warn("pullMessage exception, consumer state not ok", e); // 延迟3秒执行pull请求=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); return; } // 服务暂停 if (this.isPause()) { log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup()); // 延迟1s拉取消息=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND); return; } // 缓存消息的数量 long cachedMessageCount = processQueue.getMsgCount().get(); // 缓存消息的大小M long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); // 缓存的消息大于pull消息的阈值1000 if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) { // 50ms后拉取=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); if ((queueFlowControlTimes++ % 1000) == 0) { log.warn( "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}", this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes); } return; } // 缓存消息队列的大小大于阈值100m if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) { // 延迟50ms后拉取消息=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); if ((queueFlowControlTimes++ % 1000) == 0) { log.warn( "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}", this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes); } return; } // 顺序消费 if (!this.consumeOrderly) { // 处理队列最大数量大于消费并发最大数2000 if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) { // 50ms后拉取消息=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { log.warn( "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), pullRequest, queueMaxSpanFlowControlTimes); } return; } // 不是顺序消费 } else { if (processQueue.isLocked()) { if (!pullRequest.isLockedFirst()) { // 计算从哪里拉取消息=》 final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); boolean brokerBusy = offset < pullRequest.getNextOffset(); log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy); if (brokerBusy) { log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}", pullRequest, offset); } pullRequest.setLockedFirst(true); pullRequest.setNextOffset(offset); } } else { // 3s后拉取消息=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); log.info("pull message later because not locked in broker, {}", pullRequest); return; } } // 获取topic订阅数据 final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); if (null == subscriptionData) { // 3s后拉取消息=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); log.warn("find the consumer's subscription failed, {}", pullRequest); return; } final long beginTimestamp = System.currentTimeMillis(); // 拉去信息回调 PullCallback pullCallback = new PullCallback() { @Override public void onSuccess(PullResult pullResult) { if (pullResult != null) { // 处理拉取消息请求=》 pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, subscriptionData); switch (pullResult.getPullStatus()) { case FOUND: long prevRequestOffset = pullRequest.getNextOffset(); // 找到消息修改拉取的offset pullRequest.setNextOffset(pullResult.getNextBeginOffset()); long pullRT = System.currentTimeMillis() - beginTimestamp; // 记录topic、消费组拉取消息次数 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullRT); long firstMsgOffset = Long.MAX_VALUE; if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) { // 没找到消息立即拉取消息=》 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); } else { // 获取查询offset firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset(); // 记录topic、消费组拉取消息次数 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size()); boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); // 提交消费请求=》 DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest( pullResult.getMsgFoundList(), processQueue, pullRequest.getMessageQueue(), dispatchToConsume); // 拉取消息频次大于0 if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) { // 延迟拉取消息=》 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval()); } else { // 立即拉取消息=》 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); } } if (pullResult.getNextBeginOffset() < prevRequestOffset || firstMsgOffset < prevRequestOffset) { log.warn( "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}", pullResult.getNextBeginOffset(), firstMsgOffset, prevRequestOffset); } break; case NO_NEW_MSG: pullRequest.setNextOffset(pullResult.getNextBeginOffset()); // 修改tag offset=》 DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); // 立即执行消息拉取=》 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); break; case NO_MATCHED_MSG: pullRequest.setNextOffset(pullResult.getNextBeginOffset()); // 修改 tag offset=》 DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); // 立即消息拉取=》 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); break; case OFFSET_ILLEGAL: log.warn("the pull request offset illegal, {} {}", pullRequest.toString(), pullResult.toString()); pullRequest.setNextOffset(pullResult.getNextBeginOffset()); // offset无效,删除处理队列 pullRequest.getProcessQueue().setDropped(true); // 延迟执行任务 DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() { @Override public void run() { try { // 更新消息队列的下个offset DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), false); // 持久化消息队列=》 DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue()); // 删除消息队列=》 DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue()); log.warn("fix the pull request offset, {}", pullRequest); } catch (Throwable e) { log.error("executeTaskLater Exception", e); } } }, 10000); break; default: break; } } } @Override public void onException(Throwable e) { // 不是重试topic if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { log.warn("execute the pull request exception", e); } // 3s后执行拉取消息=》 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); } }; boolean commitOffsetEnable = false; long commitOffsetValue = 0L; // 集群消费 if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) { // 从内存中读取offset=》 commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY); if (commitOffsetValue > 0) { commitOffsetEnable = true; } } String subExpression = null; boolean classFilter = false; // 获取topic的订阅数据 SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); if (sd != null) { if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) { subExpression = sd.getSubString(); } classFilter = sd.isClassFilterMode(); } int sysFlag = PullSysFlag.buildSysFlag( commitOffsetEnable, // commitOffset true, // suspend subExpression != null, // subscription classFilter // class filter ); try { // 拉取消息=》 this.pullAPIWrapper.pullKernelImpl( pullRequest.getMessageQueue(), subExpression, subscriptionData.getExpressionType(), subscriptionData.getSubVersion(), pullRequest.getNextOffset(), this.defaultMQPushConsumer.getPullBatchSize(), sysFlag, commitOffsetValue, BROKER_SUSPEND_MAX_TIME_MILLIS, CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, CommunicationMode.ASYNC, pullCallback ); } catch (Exception e) { log.error("pullKernelImpl exception", e); // 3s后执行拉取消息=》 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); } } // private void makeSureStateOK() throws MQClientException { if (this.serviceState != ServiceState.RUNNING) { throw new MQClientException("The consumer service state not OK, " + this.serviceState + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); } } // private void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { // =》 this.mQClientFactory.getPullMessageService().executePullRequestLater(pullRequest, timeDelay); } public boolean isPause() { return pause; } public void setPause(boolean pause) { this.pause = pause; } // public ConsumerStatsManager getConsumerStatsManager() { return this.mQClientFactory.getConsumerStatsManager(); } // public void executePullRequestImmediately(final PullRequest pullRequest) { // =》 this.mQClientFactory.getPullMessageService().executePullRequestImmediately(pullRequest); } // private void correctTagsOffset(final PullRequest pullRequest) { if (0L == pullRequest.getProcessQueue().getMsgCount().get()) { // 更新消息队列的下个offset=》 this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), true); } } // public void executeTaskLater(final Runnable r, final long timeDelay) { this.mQClientFactory.getPullMessageService().executeTaskLater(r, timeDelay); } public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, InterruptedException { return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } // public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws MQClientException, InterruptedException { return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); } public void registerMessageListener(MessageListener messageListener) { this.messageListenerInner = messageListener; } // public void resume() { this.pause = false; // 负载均衡=》 doRebalance(); log.info("resume this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); } // public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { try { // 按brokerName找到master broker地址 String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); // =》 this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, // 最大消费次数,默认16 this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); } catch (Exception e) { log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e); // 创建重试topic消息 Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); String originMsgId = MessageAccessor.getOriginMessageId(msg); MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); newMsg.setFlag(msg.getFlag()); MessageAccessor.setProperties(newMsg, msg.getProperties()); MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); // 发送消息=》 this.mQClientFactory.getDefaultMQProducer().send(newMsg); } } private int getMaxReconsumeTimes() { // default reconsume times: 16 if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) { return 16; } else { return this.defaultMQPushConsumer.getMaxReconsumeTimes(); } } // public synchronized void shutdown() { switch (this.serviceState) { case CREATE_JUST: break; case RUNNING: // 消费消息服务关闭=》 this.consumeMessageService.shutdown(); // 持久化消费者offset=》 this.persistConsumerOffset(); // 消费者取消注册=》 this.mQClientFactory.unregisterConsumer(this.defaultMQPushConsumer.getConsumerGroup()); // 消费者服务关闭=》 this.mQClientFactory.shutdown(); log.info("the consumer [{}] shutdown OK", this.defaultMQPushConsumer.getConsumerGroup()); // 处理队列销毁=》 this.rebalanceImpl.destroy(); this.serviceState = ServiceState.SHUTDOWN_ALREADY; break; case SHUTDOWN_ALREADY: break; default: break; } } // 消费者启动 public synchronized void start() throws MQClientException { switch (this.serviceState) { case CREATE_JUST: log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(), this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode()); this.serviceState = ServiceState.START_FAILED; // 检查配置=》 this.checkConfig(); // copy订阅配置=》 this.copySubscription(); if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) { this.defaultMQPushConsumer.changeInstanceNameToPID(); } // 创建消费者 this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook); this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup()); this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel()); this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()); this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); this.pullAPIWrapper = new PullAPIWrapper( mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode()); // 注册过滤消息的钩子方法 this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); if (this.defaultMQPushConsumer.getOffsetStore() != null) { this.offsetStore = this.defaultMQPushConsumer.getOffsetStore(); } else { switch (this.defaultMQPushConsumer.getMessageModel()) { case BROADCASTING: // 广播 本地存储 this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup()); break; case CLUSTERING: // 集群 远程存储 this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup()); break; default: break; } this.defaultMQPushConsumer.setOffsetStore(this.offsetStore); } // 加载offset=》 this.offsetStore.load(); if (this.getMessageListenerInner() instanceof MessageListenerOrderly) { this.consumeOrderly = true; this.consumeMessageService = new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner()); } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) { this.consumeOrderly = false; this.consumeMessageService = new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner()); } // 消息消息服务启动=》 this.consumeMessageService.start(); boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this); if (!registerOK) { this.serviceState = ServiceState.CREATE_JUST; this.consumeMessageService.shutdown(); throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup() + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); } mQClientFactory.start(); log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup()); this.serviceState = ServiceState.RUNNING; break; case RUNNING: case START_FAILED: case SHUTDOWN_ALREADY: throw new MQClientException("The PushConsumer service state not OK, maybe started once, " + this.serviceState + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); default: break; } this.updateTopicSubscribeInfoWhenSubscriptionChanged(); this.mQClientFactory.checkClientInBroker(); this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); this.mQClientFactory.rebalanceImmediately(); } // private void checkConfig() throws MQClientException { Validators.checkGroup(this.defaultMQPushConsumer.getConsumerGroup()); if (null == this.defaultMQPushConsumer.getConsumerGroup()) { throw new MQClientException( "consumerGroup is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // 消费组名称不能和DEFAULT_CONSUMER一样 if (this.defaultMQPushConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { throw new MQClientException( "consumerGroup can not equal " + MixAll.DEFAULT_CONSUMER_GROUP + ", please specify another one." + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // 必须指定消息消费模式,默认集群消费 if (null == this.defaultMQPushConsumer.getMessageModel()) { throw new MQClientException( "messageModel is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // 默认从最后的offset消费 if (null == this.defaultMQPushConsumer.getConsumeFromWhere()) { throw new MQClientException( "consumeFromWhere is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // 默认消费时间半小时之前 Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYYMMDDHHMMSS); if (null == dt) { throw new MQClientException( "consumeTimestamp is invalid, the valid format is yyyyMMddHHmmss,but received " + this.defaultMQPushConsumer.getConsumeTimestamp() + " " + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // allocateMessageQueueStrategy 可以执行消息分配算法 if (null == this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()) { throw new MQClientException( "allocateMessageQueueStrategy is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // subscription if (null == this.defaultMQPushConsumer.getSubscription()) { throw new MQClientException( "subscription is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // messageListener if (null == this.defaultMQPushConsumer.getMessageListener()) { throw new MQClientException( "messageListener is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // 书否顺序消费 boolean orderly = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerOrderly; // 是否集群消费 boolean concurrently = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerConcurrently; if (!orderly && !concurrently) { throw new MQClientException( "messageListener must be instanceof MessageListenerOrderly or MessageListenerConcurrently" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeThreadMin 消费最小线程数大于1小于1000,默认20 if (this.defaultMQPushConsumer.getConsumeThreadMin() < 1 || this.defaultMQPushConsumer.getConsumeThreadMin() > 1000) { throw new MQClientException( "consumeThreadMin Out of range [1, 1000]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeThreadMax 消费最大线程数大于1小于1000,默认64 if (this.defaultMQPushConsumer.getConsumeThreadMax() < 1 || this.defaultMQPushConsumer.getConsumeThreadMax() > 1000) { throw new MQClientException( "consumeThreadMax Out of range [1, 1000]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeThreadMin can't be larger than consumeThreadMax 消费最小线程数不能大于最大线程数 if (this.defaultMQPushConsumer.getConsumeThreadMin() > this.defaultMQPushConsumer.getConsumeThreadMax()) { throw new MQClientException( "consumeThreadMin (" + this.defaultMQPushConsumer.getConsumeThreadMin() + ") " + "is larger than consumeThreadMax (" + this.defaultMQPushConsumer.getConsumeThreadMax() + ")", null); } // consumeConcurrentlyMaxSpan 并发消费最大 大于1小于65535,默认2000 if (this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() < 1 || this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() > 65535) { throw new MQClientException( "consumeConcurrentlyMaxSpan Out of range [1, 65535]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // pullThresholdForQueue 拉取消息批量大小 大于1小于65536 默认1000 if (this.defaultMQPushConsumer.getPullThresholdForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdForQueue() > 65535) { throw new MQClientException( "pullThresholdForQueue Out of range [1, 65535]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // pullThresholdForTopic if (this.defaultMQPushConsumer.getPullThresholdForTopic() != -1) { if (this.defaultMQPushConsumer.getPullThresholdForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdForTopic() > 6553500) { throw new MQClientException( "pullThresholdForTopic Out of range [1, 6553500]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } } // pullThresholdSizeForQueue 拉取消息批量大小 大于1M小于1G,默认100 if (this.defaultMQPushConsumer.getPullThresholdSizeForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForQueue() > 1024) { throw new MQClientException( "pullThresholdSizeForQueue Out of range [1, 1024]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() != -1) { // pullThresholdSizeForTopic if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForTopic() > 102400) { throw new MQClientException( "pullThresholdSizeForTopic Out of range [1, 102400]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } } // pullInterval 拉取消息频次 大于0小于65535 默认0 if (this.defaultMQPushConsumer.getPullInterval() < 0 || this.defaultMQPushConsumer.getPullInterval() > 65535) { throw new MQClientException( "pullInterval Out of range [0, 65535]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeMessageBatchMaxSize 消费消息批量最大 大于1小于1024 默认1 if (this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() < 1 || this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() > 1024) { throw new MQClientException( "consumeMessageBatchMaxSize Out of range [1, 1024]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // pullBatchSize 批量拉取大小 大于1小于1024,默认32 if (this.defaultMQPushConsumer.getPullBatchSize() < 1 || this.defaultMQPushConsumer.getPullBatchSize() > 1024) { throw new MQClientException( "pullBatchSize Out of range [1, 1024]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } } // private void copySubscription() throws MQClientException { try { Map<String, String> sub = this.defaultMQPushConsumer.getSubscription(); if (sub != null) { for (final Map.Entry<String, String> entry : sub.entrySet()) { final String topic = entry.getKey(); final String subString = entry.getValue(); // 构建订阅配置=》 SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), topic, subString); // 存储订阅配置=》 this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); } } if (null == this.messageListenerInner) { this.messageListenerInner = this.defaultMQPushConsumer.getMessageListener(); } switch (this.defaultMQPushConsumer.getMessageModel()) { // 广播 case BROADCASTING: break; // 集群消费 case CLUSTERING: // 重试topic final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()); // 构建重试topic订阅配置 SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), retryTopic, SubscriptionData.SUB_ALL); // 存储订阅配置=》 this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData); break; default: break; } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } public MessageListener getMessageListenerInner() { return messageListenerInner; } // private void updateTopicSubscribeInfoWhenSubscriptionChanged() { Map<String, SubscriptionData> subTable = this.getSubscriptionInner(); if (subTable != null) { for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) { final String topic = entry.getKey(); this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); } } } // public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() { return this.rebalanceImpl.getSubscriptionInner(); } // public void subscribe(String topic, String subExpression) throws MQClientException { try { // 构建topic订阅数据,默认集群消费=》 SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), topic, subExpression); // 存储topic订阅数据=》 this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { // 同步向所有的broker发送心跳监测=》 this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } // public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException { try { SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), topic, "*"); subscriptionData.setSubString(fullClassName); subscriptionData.setClassFilterMode(true); subscriptionData.setFilterClassSource(filterClassSource); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } // public void subscribe(final String topic, final MessageSelector messageSelector) throws MQClientException { try { if (messageSelector == null) { subscribe(topic, SubscriptionData.SUB_ALL); return; } SubscriptionData subscriptionData = FilterAPI.build(topic, messageSelector.getExpression(), messageSelector.getExpressionType()); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } // public void suspend() { this.pause = true; log.info("suspend this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); } public void unsubscribe(String topic) { this.rebalanceImpl.getSubscriptionInner().remove(topic); } public void updateConsumeOffset(MessageQueue mq, long offset) { this.offsetStore.updateOffset(mq, offset, false); } public void updateCorePoolSize(int corePoolSize) { this.consumeMessageService.updateCorePoolSize(corePoolSize); } // public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); } public RebalanceImpl getRebalanceImpl() { return rebalanceImpl; } public boolean isConsumeOrderly() { return consumeOrderly; } public void setConsumeOrderly(boolean consumeOrderly) { this.consumeOrderly = consumeOrderly; } public void resetOffsetByTimeStamp(long timeStamp) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { for (String topic : rebalanceImpl.getSubscriptionInner().keySet()) { Set<MessageQueue> mqs = rebalanceImpl.getTopicSubscribeInfoTable().get(topic); Map<MessageQueue, Long> offsetTable = new HashMap<MessageQueue, Long>(); if (mqs != null) { for (MessageQueue mq : mqs) { long offset = searchOffset(mq, timeStamp); offsetTable.put(mq, offset); } this.mQClientFactory.resetOffset(topic, groupName(), offsetTable); } } } // public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } @Override public String groupName() { return this.defaultMQPushConsumer.getConsumerGroup(); } // @Override public MessageModel messageModel() { return this.defaultMQPushConsumer.getMessageModel(); } @Override public ConsumeType consumeType() { return ConsumeType.CONSUME_PASSIVELY; } @Override public ConsumeFromWhere consumeFromWhere() { return this.defaultMQPushConsumer.getConsumeFromWhere(); } @Override public Set<SubscriptionData> subscriptions() { Set<SubscriptionData> subSet = new HashSet<SubscriptionData>(); // 获取topic订阅数据 subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values()); return subSet; } // @Override public void doRebalance() { if (!this.pause) { // =》 this.rebalanceImpl.doRebalance(this.isConsumeOrderly()); } } // @Override public void persistConsumerOffset() { try { this.makeSureStateOK(); Set<MessageQueue> mqs = new HashSet<MessageQueue>(); Set<MessageQueue> allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); mqs.addAll(allocateMq); // 持久化客户端offset=》 this.offsetStore.persistAll(mqs); } catch (Exception e) { log.error("group: " + this.defaultMQPushConsumer.getConsumerGroup() + " persistConsumerOffset exception", e); } } @Override public void updateTopicSubscribeInfo(String topic, Set<MessageQueue> info) { Map<String, SubscriptionData> subTable = this.getSubscriptionInner(); if (subTable != null) { if (subTable.containsKey(topic)) { this.rebalanceImpl.topicSubscribeInfoTable.put(topic, info); } } } @Override public boolean isSubscribeTopicNeedUpdate(String topic) { Map<String, SubscriptionData> subTable = this.getSubscriptionInner(); if (subTable != null) { if (subTable.containsKey(topic)) { return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); } } return false; } @Override public boolean isUnitMode() { return this.defaultMQPushConsumer.isUnitMode(); } @Override public ConsumerRunningInfo consumerRunningInfo() { // 消费者运行信息 ConsumerRunningInfo info = new ConsumerRunningInfo(); // MQPushConsumer信息 Properties prop = MixAll.object2Properties(this.defaultMQPushConsumer); // 消费顺序 prop.put(ConsumerRunningInfo.PROP_CONSUME_ORDERLY, String.valueOf(this.consumeOrderly)); // 消费线程池线程数 prop.put(ConsumerRunningInfo.PROP_THREADPOOL_CORE_SIZE, String.valueOf(this.consumeMessageService.getCorePoolSize())); // 消费开始时间 prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp)); info.setProperties(prop); // 订阅信息=》 Set<SubscriptionData> subSet = this.subscriptions(); info.getSubscriptionSet().addAll(subSet); Iterator<Entry<MessageQueue, ProcessQueue>> it = this.rebalanceImpl.getProcessQueueTable().entrySet().iterator(); while (it.hasNext()) { Entry<MessageQueue, ProcessQueue> next = it.next(); MessageQueue mq = next.getKey(); ProcessQueue pq = next.getValue(); // 消息处理队列 ProcessQueueInfo pqinfo = new ProcessQueueInfo(); pqinfo.setCommitOffset(this.offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE)); pq.fillProcessQueueInfo(pqinfo); info.getMqTable().put(mq, pqinfo); } for (SubscriptionData sd : subSet) { // 消费状态 ConsumeStatus consumeStatus = this.mQClientFactory.getConsumerStatsManager().consumeStatus(this.groupName(), sd.getTopic()); info.getStatusTable().put(sd.getTopic(), consumeStatus); } return info; } public MQClientInstance getmQClientFactory() { return mQClientFactory; } public void setmQClientFactory(MQClientInstance mQClientFactory) { this.mQClientFactory = mQClientFactory; } public ServiceState getServiceState() { return serviceState; } //Don't use this deprecated setter, which will be removed soon. @Deprecated public synchronized void setServiceState(ServiceState serviceState) { this.serviceState = serviceState; } // public void adjustThreadPool() { // 计算消息处理次数=》 long computeAccTotal = this.computeAccumulationTotal(); // 线程池阈值默认值100000 long adjustThreadPoolNumsThreshold = this.defaultMQPushConsumer.getAdjustThreadPoolNumsThreshold(); long incThreshold = (long) (adjustThreadPoolNumsThreshold * 1.0); long decThreshold = (long) (adjustThreadPoolNumsThreshold * 0.8); // 线程数到达增长的阈值 if (computeAccTotal >= incThreshold) { this.consumeMessageService.incCorePoolSize(); } // 线程数小于递减的阈值 if (computeAccTotal < decThreshold) { this.consumeMessageService.decCorePoolSize(); } } // private long computeAccumulationTotal() { long msgAccTotal = 0; // 处理队列 ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = this.rebalanceImpl.getProcessQueueTable(); Iterator<Entry<MessageQueue, ProcessQueue>> it = processQueueTable.entrySet().iterator(); while (it.hasNext()) { Entry<MessageQueue, ProcessQueue> next = it.next(); ProcessQueue value = next.getValue(); // 消息处理次数 msgAccTotal += value.getMsgAccCnt(); } return msgAccTotal; } public List<QueueTimeSpan> queryConsumeTimeSpan(final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { List<QueueTimeSpan> queueTimeSpan = new ArrayList<QueueTimeSpan>(); TopicRouteData routeData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000); for (BrokerData brokerData : routeData.getBrokerDatas()) { String addr = brokerData.selectBrokerAddr(); queueTimeSpan.addAll(this.mQClientFactory.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, groupName(), 3000)); } return queueTimeSpan; } public ConsumeMessageService getConsumeMessageService() { return consumeMessageService; } public void setConsumeMessageService(ConsumeMessageService consumeMessageService) { this.consumeMessageService = consumeMessageService; } }