/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.rocketmq.client.impl.consumer; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.concurrent.ConcurrentMap; import org.apache.rocketmq.client.QueryResult; import org.apache.rocketmq.client.Validators; import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; import org.apache.rocketmq.client.consumer.MessageSelector; import org.apache.rocketmq.client.consumer.PullCallback; import org.apache.rocketmq.client.consumer.PullResult; import org.apache.rocketmq.client.consumer.listener.MessageListener; import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; import org.apache.rocketmq.client.consumer.listener.MessageListenerOrderly; import org.apache.rocketmq.client.consumer.store.LocalFileOffsetStore; import org.apache.rocketmq.client.consumer.store.OffsetStore; import org.apache.rocketmq.client.consumer.store.ReadOffsetType; import org.apache.rocketmq.client.consumer.store.RemoteBrokerOffsetStore; import org.apache.rocketmq.client.exception.MQBrokerException; import org.apache.rocketmq.client.exception.MQClientException; import org.apache.rocketmq.client.hook.ConsumeMessageContext; import org.apache.rocketmq.client.hook.ConsumeMessageHook; import org.apache.rocketmq.client.hook.FilterMessageHook; import org.apache.rocketmq.client.impl.CommunicationMode; import org.apache.rocketmq.client.impl.MQClientManager; import org.apache.rocketmq.client.impl.factory.MQClientInstance; import org.apache.rocketmq.client.log.ClientLogger; import org.apache.rocketmq.client.stat.ConsumerStatsManager; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceState; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.consumer.ConsumeFromWhere; import org.apache.rocketmq.common.filter.FilterAPI; import org.apache.rocketmq.common.help.FAQUrl; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.common.message.Message; import org.apache.rocketmq.common.message.MessageAccessor; import org.apache.rocketmq.common.message.MessageConst; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageQueue; import org.apache.rocketmq.common.protocol.body.ConsumeStatus; import org.apache.rocketmq.common.protocol.body.ConsumerRunningInfo; import org.apache.rocketmq.common.protocol.body.ProcessQueueInfo; import org.apache.rocketmq.common.protocol.body.QueueTimeSpan; import org.apache.rocketmq.common.protocol.heartbeat.ConsumeType; import org.apache.rocketmq.common.protocol.heartbeat.MessageModel; import org.apache.rocketmq.common.protocol.heartbeat.SubscriptionData; import org.apache.rocketmq.common.protocol.route.BrokerData; import org.apache.rocketmq.common.protocol.route.TopicRouteData; import org.apache.rocketmq.common.sysflag.PullSysFlag; import org.apache.rocketmq.remoting.RPCHook; import org.apache.rocketmq.remoting.common.RemotingHelper; import org.apache.rocketmq.remoting.exception.RemotingException; /** * 默认的以Push为消费的方式的实现 * @author ; */ public class DefaultMQPushConsumerImpl implements MQConsumerInner { /** * Delay some time when exception occur */ private static final long PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION = 3000; /** * Flow control interval */ private static final long PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL = 50; /** * Delay some time when suspend pull service * 1分钟以后再执行继续拉取 */ private static final long PULL_TIME_DELAY_MILLS_WHEN_SUSPEND = 1000; private static final long BROKER_SUSPEND_MAX_TIME_MILLIS = 1000 * 15; private static final long CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND = 1000 * 30; private final InternalLogger log = ClientLogger.getLog(); /** * defaultMQPushConsumer */ private final DefaultMQPushConsumer defaultMQPushConsumer; /** * rebalancePushImpl */ private final RebalanceImpl rebalanceImpl = new RebalancePushImpl(this); /** * filterMessageHook */ private final ArrayList<FilterMessageHook> filterMessageHookList = new ArrayList<FilterMessageHook>(); /** * 消费的开始时间 */ private final long consumerStartTimestamp = System.currentTimeMillis(); /** * 消费Message的勾子 */ private final ArrayList<ConsumeMessageHook> consumeMessageHookList = new ArrayList<ConsumeMessageHook>(); /** * prcHook */ private final RPCHook rpcHook; /** * service的状态 */ private volatile ServiceState serviceState = ServiceState.CREATE_JUST; /** * mqClient实例 */ private MQClientInstance mQClientFactory; /** * 拉取消息的API包装 */ private PullAPIWrapper pullAPIWrapper; /** * 是否挂起 */ private volatile boolean pause = false; /** * 是否是顺序消费的 */ private boolean consumeOrderly = false; /** * MessageListener */ private MessageListener messageListenerInner; /** * 偏移量存储 */ private OffsetStore offsetStore; /** * 消费Message的Service */ private ConsumeMessageService consumeMessageService; /** * 队列流控次数 */ private long queueFlowControlTimes = 0; /** * 队列流控相关参数 */ private long queueMaxSpanFlowControlTimes = 0; /** * 构造函数 */ public DefaultMQPushConsumerImpl(DefaultMQPushConsumer defaultMQPushConsumer, RPCHook rpcHook) { this.defaultMQPushConsumer = defaultMQPushConsumer; this.rpcHook = rpcHook; } /** * 注册FilterMessageHook */ public void registerFilterMessageHook(final FilterMessageHook hook) { this.filterMessageHookList.add(hook); log.info("register FilterMessageHook Hook, {}", hook.hookName()); } //====hook相关 public boolean hasHook() { return !this.consumeMessageHookList.isEmpty(); } public void registerConsumeMessageHook(final ConsumeMessageHook hook) { this.consumeMessageHookList.add(hook); log.info("register consumeMessageHook Hook, {}", hook.hookName()); } public void executeHookBefore(final ConsumeMessageContext context) { if (!this.consumeMessageHookList.isEmpty()) { for (ConsumeMessageHook hook : this.consumeMessageHookList) { try { hook.consumeMessageBefore(context); } catch (Throwable e) { } } } } public void executeHookAfter(final ConsumeMessageContext context) { if (!this.consumeMessageHookList.isEmpty()) { for (ConsumeMessageHook hook : this.consumeMessageHookList) { try { hook.consumeMessageAfter(context); } catch (Throwable e) { } } } } //===========hook相关end //============ 重复代码 start public void createTopic(String key, String newTopic, int queueNum) throws MQClientException { createTopic(key, newTopic, queueNum, 0); } public void createTopic(String key, String newTopic, int queueNum, int topicSysFlag) throws MQClientException { this.mQClientFactory.getMQAdminImpl().createTopic(key, newTopic, queueNum, topicSysFlag); } public Set<MessageQueue> fetchSubscribeMessageQueues(String topic) throws MQClientException { Set<MessageQueue> result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); if (null == result) { this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); result = this.rebalanceImpl.getTopicSubscribeInfoTable().get(topic); } if (null == result) { throw new MQClientException("The topic[" + topic + "] not exist", null); } return result; } public DefaultMQPushConsumer getDefaultMQPushConsumer() { return defaultMQPushConsumer; } public long earliestMsgStoreTime(MessageQueue mq) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().earliestMsgStoreTime(mq); } public long maxOffset(MessageQueue mq) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().maxOffset(mq); } public long minOffset(MessageQueue mq) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().minOffset(mq); } public OffsetStore getOffsetStore() { return offsetStore; } public void setOffsetStore(OffsetStore offsetStore) { this.offsetStore = offsetStore; } //=================end /** * 被定时任务调度的PullRequest * @param pullRequest ; */ public void pullMessage(final PullRequest pullRequest) { final ProcessQueue processQueue = pullRequest.getProcessQueue(); if (processQueue.isDropped()) { log.info("the pull request[{}] is dropped.", pullRequest.toString()); return; } //设置上一次拉取时间为当前时间 pullRequest.getProcessQueue().setLastPullTimestamp(System.currentTimeMillis()); try { this.makeSureStateOK(); } catch (MQClientException e) { log.warn("pullMessage exception, consumer state not ok", e); this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); return; } //如果是暂停状态,1分钟以后再拉取 if (this.isPause()) { log.warn("consumer was paused, execute pull request later. instanceName={}, group={}", this.defaultMQPushConsumer.getInstanceName(), this.defaultMQPushConsumer.getConsumerGroup()); this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_SUSPEND); return; } //缓存的消息数量 long cachedMessageCount = processQueue.getMsgCount().get(); //缓存的消息Mb long cachedMessageSizeInMiB = processQueue.getMsgSize().get() / (1024 * 1024); //如果积压的消息比较多,就执行流控 if (cachedMessageCount > this.defaultMQPushConsumer.getPullThresholdForQueue()) { this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); if ((queueFlowControlTimes++ % 1000) == 0) { log.warn( "the cached message count exceeds the threshold {}, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}", this.defaultMQPushConsumer.getPullThresholdForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes); } return; } //如果积压的消息超过了一个配置,默认是100M,也触发流控 if (cachedMessageSizeInMiB > this.defaultMQPushConsumer.getPullThresholdSizeForQueue()) { this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); if ((queueFlowControlTimes++ % 1000) == 0) { log.warn( "the cached message size exceeds the threshold {} MiB, so do flow control, minOffset={}, maxOffset={}, count={}, size={} MiB, pullRequest={}, flowControlTimes={}", this.defaultMQPushConsumer.getPullThresholdSizeForQueue(), processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), cachedMessageCount, cachedMessageSizeInMiB, pullRequest, queueFlowControlTimes); } return; } //如果是非顺序消费 if (!this.consumeOrderly) { //如果offset跨度达到阀值,则触发流控 if (processQueue.getMaxSpan() > this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan()) { this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_FLOW_CONTROL); if ((queueMaxSpanFlowControlTimes++ % 1000) == 0) { log.warn( "the queue's messages, span too long, so do flow control, minOffset={}, maxOffset={}, maxSpan={}, pullRequest={}, flowControlTimes={}", processQueue.getMsgTreeMap().firstKey(), processQueue.getMsgTreeMap().lastKey(), processQueue.getMaxSpan(), pullRequest, queueMaxSpanFlowControlTimes); } return; } //顺序消费 } else { //是否是第一次锁定 if (processQueue.isLocked()) { //不是第一此锁定 if (!pullRequest.isLockedFirst()) { //计算从哪里拉取的偏移量 final long offset = this.rebalanceImpl.computePullFromWhere(pullRequest.getMessageQueue()); boolean brokerBusy = offset < pullRequest.getNextOffset(); log.info("the first time to pull message, so fix offset from broker. pullRequest: {} NewOffset: {} brokerBusy: {}", pullRequest, offset, brokerBusy); if (brokerBusy) { log.info("[NOTIFYME]the first time to pull message, but pull request offset larger than broker consume offset. pullRequest: {} NewOffset: {}", pullRequest, offset); } //第一次锁定为true pullRequest.setLockedFirst(true); pullRequest.setNextOffset(offset); } } else { //流控 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); log.info("pull message later because not locked in broker, {}", pullRequest); return; } } final SubscriptionData subscriptionData = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); if (null == subscriptionData) { //没有找到subscriptionData,流控 this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); log.warn("find the consumer's subscription failed, {}", pullRequest); return; } //消费开始时间 final long beginTimestamp = System.currentTimeMillis(); /* * 拉取的PullCallback */ PullCallback pullCallback = new PullCallback() { @Override public void onSuccess(PullResult pullResult) { if (pullResult != null) { pullResult = DefaultMQPushConsumerImpl.this.pullAPIWrapper.processPullResult(pullRequest.getMessageQueue(), pullResult, subscriptionData); switch (pullResult.getPullStatus()) { case FOUND: long prevRequestOffset = pullRequest.getNextOffset(); pullRequest.setNextOffset(pullResult.getNextBeginOffset()); long pullRT = System.currentTimeMillis() - beginTimestamp; //更新消费RT DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullRT(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullRT); long firstMsgOffset = Long.MAX_VALUE; if (pullResult.getMsgFoundList() == null || pullResult.getMsgFoundList().isEmpty()) { //结果为空,触发一次拉取 DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); } else { //第一条消息的偏移量 firstMsgOffset = pullResult.getMsgFoundList().get(0).getQueueOffset(); //监控项 DefaultMQPushConsumerImpl.this.getConsumerStatsManager().incPullTPS(pullRequest.getConsumerGroup(), pullRequest.getMessageQueue().getTopic(), pullResult.getMsgFoundList().size()); //处理队列添加拉取到的消息 boolean dispatchToConsume = processQueue.putMessage(pullResult.getMsgFoundList()); //consumeMessageService 提交请求 DefaultMQPushConsumerImpl.this.consumeMessageService.submitConsumeRequest( pullResult.getMsgFoundList(), processQueue, pullRequest.getMessageQueue(), dispatchToConsume); //继续拉取消息 if (DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval() > 0) { DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, DefaultMQPushConsumerImpl.this.defaultMQPushConsumer.getPullInterval()); } else { DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); } } //数据可能有错误 if (pullResult.getNextBeginOffset() < prevRequestOffset || firstMsgOffset < prevRequestOffset) { log.warn( "[BUG] pull message result maybe data wrong, nextBeginOffset: {} firstMsgOffset: {} prevRequestOffset: {}", pullResult.getNextBeginOffset(), firstMsgOffset, prevRequestOffset); } break; case NO_NEW_MSG: pullRequest.setNextOffset(pullResult.getNextBeginOffset()); DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); break; case NO_MATCHED_MSG: pullRequest.setNextOffset(pullResult.getNextBeginOffset()); DefaultMQPushConsumerImpl.this.correctTagsOffset(pullRequest); DefaultMQPushConsumerImpl.this.executePullRequestImmediately(pullRequest); break; case OFFSET_ILLEGAL: log.warn("the pull request offset illegal, {} {}", pullRequest.toString(), pullResult.toString()); pullRequest.setNextOffset(pullResult.getNextBeginOffset()); //偏移量不合法,将getProcessQueue Dropped pullRequest.getProcessQueue().setDropped(true); //10秒钟以后调度以下任务:offsetStore更新offset,并持久化。并移除getMessageQueue DefaultMQPushConsumerImpl.this.executeTaskLater(new Runnable() { @Override public void run() { try { DefaultMQPushConsumerImpl.this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), false); DefaultMQPushConsumerImpl.this.offsetStore.persist(pullRequest.getMessageQueue()); DefaultMQPushConsumerImpl.this.rebalanceImpl.removeProcessQueue(pullRequest.getMessageQueue()); log.warn("fix the pull request offset, {}", pullRequest); } catch (Throwable e) { log.error("executeTaskLater Exception", e); } } }, 10000); break; default: break; } } } @Override public void onException(Throwable e) { if (!pullRequest.getMessageQueue().getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) { log.warn("execute the pull request exception", e); } //异常以后稍后在进行拉取消息 DefaultMQPushConsumerImpl.this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); } }; boolean commitOffsetEnable = false; long commitOffsetValue = 0L; //如果是集群消费模式 if (MessageModel.CLUSTERING == this.defaultMQPushConsumer.getMessageModel()) { commitOffsetValue = this.offsetStore.readOffset(pullRequest.getMessageQueue(), ReadOffsetType.READ_FROM_MEMORY); if (commitOffsetValue > 0) { commitOffsetEnable = true; } } String subExpression = null; boolean classFilter = false; SubscriptionData sd = this.rebalanceImpl.getSubscriptionInner().get(pullRequest.getMessageQueue().getTopic()); if (sd != null) { if (this.defaultMQPushConsumer.isPostSubscriptionWhenPull() && !sd.isClassFilterMode()) { subExpression = sd.getSubString(); } classFilter = sd.isClassFilterMode(); } int sysFlag = PullSysFlag.buildSysFlag( commitOffsetEnable, // commitOffset true, // suspend subExpression != null, // subscription classFilter // class filter ); try { this.pullAPIWrapper.pullKernelImpl( pullRequest.getMessageQueue(), subExpression, subscriptionData.getExpressionType(), subscriptionData.getSubVersion(), pullRequest.getNextOffset(), this.defaultMQPushConsumer.getPullBatchSize(), sysFlag, commitOffsetValue, BROKER_SUSPEND_MAX_TIME_MILLIS, CONSUMER_TIMEOUT_MILLIS_WHEN_SUSPEND, CommunicationMode.ASYNC, pullCallback ); } catch (Exception e) { log.error("pullKernelImpl exception", e); this.executePullRequestLater(pullRequest, PULL_TIME_DELAY_MILLS_WHEN_EXCEPTION); } } /** * 确保状态ok * @throws MQClientException ; */ private void makeSureStateOK() throws MQClientException { if (this.serviceState != ServiceState.RUNNING) { throw new MQClientException("The consumer service state not OK, " + this.serviceState + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); } } /** * 稍后再执行拉取任务 * @param pullRequest ; * @param timeDelay ; */ private void executePullRequestLater(final PullRequest pullRequest, final long timeDelay) { this.mQClientFactory.getPullMessageService().executePullRequestLater(pullRequest, timeDelay); } public boolean isPause() { return pause; } public void setPause(boolean pause) { this.pause = pause; } public ConsumerStatsManager getConsumerStatsManager() { return this.mQClientFactory.getConsumerStatsManager(); } /** * 被rebanalce调用。开始触发第一次拉取消息 * @param pullRequest pullRequest */ public void executePullRequestImmediately(final PullRequest pullRequest) { this.mQClientFactory.getPullMessageService().executePullRequestImmediately(pullRequest); } private void correctTagsOffset(final PullRequest pullRequest) { if (0L == pullRequest.getProcessQueue().getMsgCount().get()) { this.offsetStore.updateOffset(pullRequest.getMessageQueue(), pullRequest.getNextOffset(), true); } } public void executeTaskLater(final Runnable r, final long timeDelay) { this.mQClientFactory.getPullMessageService().executeTaskLater(r, timeDelay); } //============ public QueryResult queryMessage(String topic, String key, int maxNum, long begin, long end) throws MQClientException, InterruptedException { return this.mQClientFactory.getMQAdminImpl().queryMessage(topic, key, maxNum, begin, end); } public MessageExt queryMessageByUniqKey(String topic, String uniqKey) throws MQClientException, InterruptedException { return this.mQClientFactory.getMQAdminImpl().queryMessageByUniqKey(topic, uniqKey); } public void registerMessageListener(MessageListener messageListener) { this.messageListenerInner = messageListener; } //================== /** * 恢复消费 */ public void resume() { this.pause = false; doRebalance(); log.info("resume this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); } /** * 将消息发回broker * @param msg ; * @param delayLevel ; * @param brokerName ; * @throws RemotingException ; * @throws MQBrokerException ; * @throws InterruptedException ; * @throws MQClientException ; */ public void sendMessageBack(MessageExt msg, int delayLevel, final String brokerName) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { try { String brokerAddr = (null != brokerName) ? this.mQClientFactory.findBrokerAddressInPublish(brokerName) : RemotingHelper.parseSocketAddressAddr(msg.getStoreHost()); this.mQClientFactory.getMQClientAPIImpl().consumerSendMessageBack(brokerAddr, msg, this.defaultMQPushConsumer.getConsumerGroup(), delayLevel, 5000, getMaxReconsumeTimes()); } catch (Exception e) { log.error("sendMessageBack Exception, " + this.defaultMQPushConsumer.getConsumerGroup(), e); Message newMsg = new Message(MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()), msg.getBody()); String originMsgId = MessageAccessor.getOriginMessageId(msg); MessageAccessor.setOriginMessageId(newMsg, UtilAll.isBlank(originMsgId) ? msg.getMsgId() : originMsgId); newMsg.setFlag(msg.getFlag()); MessageAccessor.setProperties(newMsg, msg.getProperties()); MessageAccessor.putProperty(newMsg, MessageConst.PROPERTY_RETRY_TOPIC, msg.getTopic()); MessageAccessor.setReconsumeTime(newMsg, String.valueOf(msg.getReconsumeTimes() + 1)); MessageAccessor.setMaxReconsumeTimes(newMsg, String.valueOf(getMaxReconsumeTimes())); newMsg.setDelayTimeLevel(3 + msg.getReconsumeTimes()); this.mQClientFactory.getDefaultMQProducer().send(newMsg); } } /** * 获取消息的最大的重试的次数 * @return ; */ private int getMaxReconsumeTimes() { // default reconsume times: 16 if (this.defaultMQPushConsumer.getMaxReconsumeTimes() == -1) { return 16; } else { return this.defaultMQPushConsumer.getMaxReconsumeTimes(); } } /** * shotdown时,持久化消费偏移量 */ public synchronized void shutdown() { switch (this.serviceState) { case CREATE_JUST: break; case RUNNING: this.consumeMessageService.shutdown(); //持久化消费偏移量 this.persistConsumerOffset(); this.mQClientFactory.unregisterConsumer(this.defaultMQPushConsumer.getConsumerGroup()); this.mQClientFactory.shutdown(); log.info("the consumer [{}] shutdown OK", this.defaultMQPushConsumer.getConsumerGroup()); this.rebalanceImpl.destroy(); this.serviceState = ServiceState.SHUTDOWN_ALREADY; break; case SHUTDOWN_ALREADY: break; default: break; } } /** * 启动DefaultMQPushConsumer * @throws MQClientException ; */ public synchronized void start() throws MQClientException { switch (this.serviceState) { case CREATE_JUST: log.info("the consumer [{}] start beginning. messageModel={}, isUnitMode={}", this.defaultMQPushConsumer.getConsumerGroup(), this.defaultMQPushConsumer.getMessageModel(), this.defaultMQPushConsumer.isUnitMode()); this.serviceState = ServiceState.START_FAILED; this.checkConfig(); this.copySubscription(); //如果是集群消费,changeInstanceNameToPid if (this.defaultMQPushConsumer.getMessageModel() == MessageModel.CLUSTERING) { this.defaultMQPushConsumer.changeInstanceNameToPID(); } //获取mqClientFactory this.mQClientFactory = MQClientManager.getInstance().getAndCreateMQClientInstance(this.defaultMQPushConsumer, this.rpcHook); //设置rebalanceImpl相关的参数 this.rebalanceImpl.setConsumerGroup(this.defaultMQPushConsumer.getConsumerGroup()); this.rebalanceImpl.setMessageModel(this.defaultMQPushConsumer.getMessageModel()); this.rebalanceImpl.setAllocateMessageQueueStrategy(this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()); this.rebalanceImpl.setmQClientFactory(this.mQClientFactory); //PullAPIWrapper this.pullAPIWrapper = new PullAPIWrapper( mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup(), isUnitMode()); this.pullAPIWrapper.registerFilterMessageHook(filterMessageHookList); //offsetStore if (this.defaultMQPushConsumer.getOffsetStore() != null) { this.offsetStore = this.defaultMQPushConsumer.getOffsetStore(); } else { switch (this.defaultMQPushConsumer.getMessageModel()) { //广播消息,LocalFileOffsetStore case BROADCASTING: this.offsetStore = new LocalFileOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup()); break; //集群消费 case CLUSTERING: this.offsetStore = new RemoteBrokerOffsetStore(this.mQClientFactory, this.defaultMQPushConsumer.getConsumerGroup()); break; default: break; } this.defaultMQPushConsumer.setOffsetStore(this.offsetStore); } this.offsetStore.load(); //如果是顺序消费,就标记顺序消费为true if (this.getMessageListenerInner() instanceof MessageListenerOrderly) { this.consumeOrderly = true; //顺序消费 this.consumeMessageService = new ConsumeMessageOrderlyService(this, (MessageListenerOrderly) this.getMessageListenerInner()); //并发消费 } else if (this.getMessageListenerInner() instanceof MessageListenerConcurrently) { this.consumeOrderly = false; this.consumeMessageService = new ConsumeMessageConcurrentlyService(this, (MessageListenerConcurrently) this.getMessageListenerInner()); } //消费Message的Service启动 this.consumeMessageService.start(); boolean registerOK = mQClientFactory.registerConsumer(this.defaultMQPushConsumer.getConsumerGroup(), this); if (!registerOK) { this.serviceState = ServiceState.CREATE_JUST; this.consumeMessageService.shutdown(); throw new MQClientException("The consumer group[" + this.defaultMQPushConsumer.getConsumerGroup() + "] has been created before, specify another name please." + FAQUrl.suggestTodo(FAQUrl.GROUP_NAME_DUPLICATE_URL), null); } mQClientFactory.start(); log.info("the consumer [{}] start OK.", this.defaultMQPushConsumer.getConsumerGroup()); this.serviceState = ServiceState.RUNNING; break; case RUNNING: case START_FAILED: case SHUTDOWN_ALREADY: throw new MQClientException("The PushConsumer service state not OK, maybe started once, " + this.serviceState + FAQUrl.suggestTodo(FAQUrl.CLIENT_SERVICE_NOT_OK), null); default: break; } //更新topicSubscribeInfo,当消息有变化 this.updateTopicSubscribeInfoWhenSubscriptionChanged(); //在服务端检查本consumer的消息过滤相关配置是否正确 this.mQClientFactory.checkClientInBroker(); //心跳 this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); //立即执行rebalance this.mQClientFactory.rebalanceImmediately(); } /** * 检查各种配置 * @throws MQClientException ; */ private void checkConfig() throws MQClientException { Validators.checkGroup(this.defaultMQPushConsumer.getConsumerGroup()); if (null == this.defaultMQPushConsumer.getConsumerGroup()) { throw new MQClientException( "consumerGroup is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } if (this.defaultMQPushConsumer.getConsumerGroup().equals(MixAll.DEFAULT_CONSUMER_GROUP)) { throw new MQClientException( "consumerGroup can not equal " + MixAll.DEFAULT_CONSUMER_GROUP + ", please specify another one." + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } if (null == this.defaultMQPushConsumer.getMessageModel()) { throw new MQClientException( "messageModel is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } if (null == this.defaultMQPushConsumer.getConsumeFromWhere()) { throw new MQClientException( "consumeFromWhere is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } Date dt = UtilAll.parseDate(this.defaultMQPushConsumer.getConsumeTimestamp(), UtilAll.YYYYMMDDHHMMSS); if (null == dt) { throw new MQClientException( "consumeTimestamp is invalid, the valid format is yyyyMMddHHmmss,but received " + this.defaultMQPushConsumer.getConsumeTimestamp() + " " + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // allocateMessageQueueStrategy if (null == this.defaultMQPushConsumer.getAllocateMessageQueueStrategy()) { throw new MQClientException( "allocateMessageQueueStrategy is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // subscription if (null == this.defaultMQPushConsumer.getSubscription()) { throw new MQClientException( "subscription is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // messageListener if (null == this.defaultMQPushConsumer.getMessageListener()) { throw new MQClientException( "messageListener is null" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } boolean orderly = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerOrderly; boolean concurrently = this.defaultMQPushConsumer.getMessageListener() instanceof MessageListenerConcurrently; if (!orderly && !concurrently) { throw new MQClientException( "messageListener must be instanceof MessageListenerOrderly or MessageListenerConcurrently" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeThreadMin if (this.defaultMQPushConsumer.getConsumeThreadMin() < 1 || this.defaultMQPushConsumer.getConsumeThreadMin() > 1000) { throw new MQClientException( "consumeThreadMin Out of range [1, 1000]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeThreadMax if (this.defaultMQPushConsumer.getConsumeThreadMax() < 1 || this.defaultMQPushConsumer.getConsumeThreadMax() > 1000) { throw new MQClientException( "consumeThreadMax Out of range [1, 1000]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeThreadMin can't be larger than consumeThreadMax if (this.defaultMQPushConsumer.getConsumeThreadMin() > this.defaultMQPushConsumer.getConsumeThreadMax()) { throw new MQClientException( "consumeThreadMin (" + this.defaultMQPushConsumer.getConsumeThreadMin() + ") " + "is larger than consumeThreadMax (" + this.defaultMQPushConsumer.getConsumeThreadMax() + ")", null); } // consumeConcurrentlyMaxSpan if (this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() < 1 || this.defaultMQPushConsumer.getConsumeConcurrentlyMaxSpan() > 65535) { throw new MQClientException( "consumeConcurrentlyMaxSpan Out of range [1, 65535]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // pullThresholdForQueue if (this.defaultMQPushConsumer.getPullThresholdForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdForQueue() > 65535) { throw new MQClientException( "pullThresholdForQueue Out of range [1, 65535]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // pullThresholdForTopic if (this.defaultMQPushConsumer.getPullThresholdForTopic() != -1) { if (this.defaultMQPushConsumer.getPullThresholdForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdForTopic() > 6553500) { throw new MQClientException( "pullThresholdForTopic Out of range [1, 6553500]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } } // pullThresholdSizeForQueue if (this.defaultMQPushConsumer.getPullThresholdSizeForQueue() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForQueue() > 1024) { throw new MQClientException( "pullThresholdSizeForQueue Out of range [1, 1024]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() != -1) { // pullThresholdSizeForTopic if (this.defaultMQPushConsumer.getPullThresholdSizeForTopic() < 1 || this.defaultMQPushConsumer.getPullThresholdSizeForTopic() > 102400) { throw new MQClientException( "pullThresholdSizeForTopic Out of range [1, 102400]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } } // pullInterval if (this.defaultMQPushConsumer.getPullInterval() < 0 || this.defaultMQPushConsumer.getPullInterval() > 65535) { throw new MQClientException( "pullInterval Out of range [0, 65535]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // consumeMessageBatchMaxSize if (this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() < 1 || this.defaultMQPushConsumer.getConsumeMessageBatchMaxSize() > 1024) { throw new MQClientException( "consumeMessageBatchMaxSize Out of range [1, 1024]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } // pullBatchSize if (this.defaultMQPushConsumer.getPullBatchSize() < 1 || this.defaultMQPushConsumer.getPullBatchSize() > 1024) { throw new MQClientException( "pullBatchSize Out of range [1, 1024]" + FAQUrl.suggestTodo(FAQUrl.CLIENT_PARAMETER_CHECK_URL), null); } } /** * 拷贝copySubscription * @throws MQClientException ; */ private void copySubscription() throws MQClientException { try { Map<String, String> sub = this.defaultMQPushConsumer.getSubscription(); if (sub != null) { for (final Map.Entry<String, String> entry : sub.entrySet()) { final String topic = entry.getKey(); final String subString = entry.getValue(); SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), topic, subString); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); } } //构造MessageListener if (null == this.messageListenerInner) { this.messageListenerInner = this.defaultMQPushConsumer.getMessageListener(); } switch (this.defaultMQPushConsumer.getMessageModel()) { //广播消费 case BROADCASTING: break; //集群消费 case CLUSTERING: //获取某一个组的retryTopic final String retryTopic = MixAll.getRetryTopic(this.defaultMQPushConsumer.getConsumerGroup()); SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), retryTopic, SubscriptionData.SUB_ALL); this.rebalanceImpl.getSubscriptionInner().put(retryTopic, subscriptionData); break; default: break; } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } public MessageListener getMessageListenerInner() { return messageListenerInner; } /** * 更新topicSubscribeInfoWhenSubScriptionChanged */ private void updateTopicSubscribeInfoWhenSubscriptionChanged() { Map<String, SubscriptionData> subTable = this.getSubscriptionInner(); if (subTable != null) { for (final Map.Entry<String, SubscriptionData> entry : subTable.entrySet()) { final String topic = entry.getKey(); this.mQClientFactory.updateTopicRouteInfoFromNameServer(topic); } } } /** * 获取getSubscriptionInner * @return ; */ public ConcurrentMap<String, SubscriptionData> getSubscriptionInner() { return this.rebalanceImpl.getSubscriptionInner(); } /** * subscribe ; * @param topic ; * @param subExpression ; * @throws MQClientException ; */ public void subscribe(String topic, String subExpression) throws MQClientException { try { SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), topic, subExpression); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } /** * 关注 topic。上传源代码 * @param topic ; * @param fullClassName ; * @param filterClassSource ; * @throws MQClientException ; */ public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException { try { SubscriptionData subscriptionData = FilterAPI.buildSubscriptionData(this.defaultMQPushConsumer.getConsumerGroup(), topic, "*"); subscriptionData.setSubString(fullClassName); subscriptionData.setClassFilterMode(true); subscriptionData.setFilterClassSource(filterClassSource); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } /** * subsicribe * @param topic topic * @param messageSelector 消息选择器 * @throws MQClientException ; */ public void subscribe(final String topic, final MessageSelector messageSelector) throws MQClientException { try { if (messageSelector == null) { subscribe(topic, SubscriptionData.SUB_ALL); return; } SubscriptionData subscriptionData = FilterAPI.build(topic, messageSelector.getExpression(), messageSelector.getExpressionType()); this.rebalanceImpl.getSubscriptionInner().put(topic, subscriptionData); if (this.mQClientFactory != null) { this.mQClientFactory.sendHeartbeatToAllBrokerWithLock(); } } catch (Exception e) { throw new MQClientException("subscription exception", e); } } /** * 挂起消费者 */ public void suspend() { this.pause = true; log.info("suspend this consumer, {}", this.defaultMQPushConsumer.getConsumerGroup()); } /** * 取消注册对某个consumer的监听 */ public void unsubscribe(String topic) { this.rebalanceImpl.getSubscriptionInner().remove(topic); } /** * 更新消费者队列的偏移量 * @param mq mq * @param offset offset */ public void updateConsumeOffset(MessageQueue mq, long offset) { this.offsetStore.updateOffset(mq, offset, false); } /** * 更新消费Service的核心线程数 * @param corePoolSize 核心线程数 */ public void updateCorePoolSize(int corePoolSize) { this.consumeMessageService.updateCorePoolSize(corePoolSize); } /** * 查询Message * @param msgId msgid * @return ; * @throws RemotingException ; * @throws MQBrokerException ; * @throws InterruptedException ; * @throws MQClientException ; */ public MessageExt viewMessage(String msgId) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { return this.mQClientFactory.getMQAdminImpl().viewMessage(msgId); } public RebalanceImpl getRebalanceImpl() { return rebalanceImpl; } public boolean isConsumeOrderly() { return consumeOrderly; } public void setConsumeOrderly(boolean consumeOrderly) { this.consumeOrderly = consumeOrderly; } public void resetOffsetByTimeStamp(long timeStamp) throws RemotingException, MQBrokerException, InterruptedException, MQClientException { for (String topic : rebalanceImpl.getSubscriptionInner().keySet()) { Set<MessageQueue> mqs = rebalanceImpl.getTopicSubscribeInfoTable().get(topic); Map<MessageQueue, Long> offsetTable = new HashMap<MessageQueue, Long>(); if (mqs != null) { for (MessageQueue mq : mqs) { long offset = searchOffset(mq, timeStamp); offsetTable.put(mq, offset); } this.mQClientFactory.resetOffset(topic, groupName(), offsetTable); } } } /** * 查询某个队列的偏移量 * @param mq mq * @param timestamp ; * @return ; * @throws MQClientException ; */ public long searchOffset(MessageQueue mq, long timestamp) throws MQClientException { return this.mQClientFactory.getMQAdminImpl().searchOffset(mq, timestamp); } @Override public String groupName() { return this.defaultMQPushConsumer.getConsumerGroup(); } @Override public MessageModel messageModel() { return this.defaultMQPushConsumer.getMessageModel(); } @Override public ConsumeType consumeType() { return ConsumeType.CONSUME_PASSIVELY; } @Override public ConsumeFromWhere consumeFromWhere() { return this.defaultMQPushConsumer.getConsumeFromWhere(); } /** * 获取全部的SubscriptionData,作为心跳数据 * @return ; */ @Override public Set<SubscriptionData> subscriptions() { Set<SubscriptionData> subSet = new HashSet<SubscriptionData>(); subSet.addAll(this.rebalanceImpl.getSubscriptionInner().values()); return subSet; } /** * 执行rebalance */ @Override public void doRebalance() { if (!this.pause) { this.rebalanceImpl.doRebalance(this.isConsumeOrderly()); } } /** * 持久化消费偏移量 */ @Override public void persistConsumerOffset() { try { this.makeSureStateOK(); Set<MessageQueue> mqs = new HashSet<MessageQueue>(); Set<MessageQueue> allocateMq = this.rebalanceImpl.getProcessQueueTable().keySet(); mqs.addAll(allocateMq); this.offsetStore.persistAll(mqs); } catch (Exception e) { log.error("group: " + this.defaultMQPushConsumer.getConsumerGroup() + " persistConsumerOffset exception", e); } } /** * 更新topic 订阅的信息 * @param topic topic * @param info 队列 */ @Override public void updateTopicSubscribeInfo(String topic, Set<MessageQueue> info) { Map<String, SubscriptionData> subTable = this.getSubscriptionInner(); if (subTable != null) { if (subTable.containsKey(topic)) { this.rebalanceImpl.topicSubscribeInfoTable.put(topic, info); } } } /** * 是否需要更新 * @param topic topic * @return ; */ @Override public boolean isSubscribeTopicNeedUpdate(String topic) { Map<String, SubscriptionData> subTable = this.getSubscriptionInner(); if (subTable != null) { if (subTable.containsKey(topic)) { return !this.rebalanceImpl.topicSubscribeInfoTable.containsKey(topic); } } return false; } @Override public boolean isUnitMode() { return this.defaultMQPushConsumer.isUnitMode(); } /** * 获取消费者的运行时信息 * @return ; */ @Override public ConsumerRunningInfo consumerRunningInfo() { ConsumerRunningInfo info = new ConsumerRunningInfo(); Properties prop = MixAll.object2Properties(this.defaultMQPushConsumer); prop.put(ConsumerRunningInfo.PROP_CONSUME_ORDERLY, String.valueOf(this.consumeOrderly)); prop.put(ConsumerRunningInfo.PROP_THREADPOOL_CORE_SIZE, String.valueOf(this.consumeMessageService.getCorePoolSize())); prop.put(ConsumerRunningInfo.PROP_CONSUMER_START_TIMESTAMP, String.valueOf(this.consumerStartTimestamp)); info.setProperties(prop); Set<SubscriptionData> subSet = this.subscriptions(); info.getSubscriptionSet().addAll(subSet); Iterator<Entry<MessageQueue, ProcessQueue>> it = this.rebalanceImpl.getProcessQueueTable().entrySet().iterator(); while (it.hasNext()) { Entry<MessageQueue, ProcessQueue> next = it.next(); MessageQueue mq = next.getKey(); ProcessQueue pq = next.getValue(); ProcessQueueInfo pqinfo = new ProcessQueueInfo(); pqinfo.setCommitOffset(this.offsetStore.readOffset(mq, ReadOffsetType.MEMORY_FIRST_THEN_STORE)); pq.fillProcessQueueInfo(pqinfo); info.getMqTable().put(mq, pqinfo); } for (SubscriptionData sd : subSet) { ConsumeStatus consumeStatus = this.mQClientFactory.getConsumerStatsManager().consumeStatus(this.groupName(), sd.getTopic()); info.getStatusTable().put(sd.getTopic(), consumeStatus); } return info; } public MQClientInstance getmQClientFactory() { return mQClientFactory; } public void setmQClientFactory(MQClientInstance mQClientFactory) { this.mQClientFactory = mQClientFactory; } public ServiceState getServiceState() { return serviceState; } //Don't use this deprecated setter, which will be removed soon. @Deprecated public synchronized void setServiceState(ServiceState serviceState) { this.serviceState = serviceState; } /** * 调整线程池 */ public void adjustThreadPool() { long computeAccTotal = this.computeAccumulationTotal(); long adjustThreadPoolNumsThreshold = this.defaultMQPushConsumer.getAdjustThreadPoolNumsThreshold(); long incThreshold = (long) (adjustThreadPoolNumsThreshold * 1.0); long decThreshold = (long) (adjustThreadPoolNumsThreshold * 0.8); if (computeAccTotal >= incThreshold) { this.consumeMessageService.incCorePoolSize(); } if (computeAccTotal < decThreshold) { this.consumeMessageService.decCorePoolSize(); } } /** * 获取总共接受的消息的数量 * @return ; */ private long computeAccumulationTotal() { long msgAccTotal = 0; ConcurrentMap<MessageQueue, ProcessQueue> processQueueTable = this.rebalanceImpl.getProcessQueueTable(); Iterator<Entry<MessageQueue, ProcessQueue>> it = processQueueTable.entrySet().iterator(); while (it.hasNext()) { Entry<MessageQueue, ProcessQueue> next = it.next(); ProcessQueue value = next.getValue(); msgAccTotal += value.getMsgAccCnt(); } return msgAccTotal; } public List<QueueTimeSpan> queryConsumeTimeSpan(final String topic) throws RemotingException, MQClientException, InterruptedException, MQBrokerException { List<QueueTimeSpan> queueTimeSpan = new ArrayList<QueueTimeSpan>(); TopicRouteData routeData = this.mQClientFactory.getMQClientAPIImpl().getTopicRouteInfoFromNameServer(topic, 3000); for (BrokerData brokerData : routeData.getBrokerDatas()) { String addr = brokerData.selectBrokerAddr(); queueTimeSpan.addAll(this.mQClientFactory.getMQClientAPIImpl().queryConsumeTimeSpan(addr, topic, groupName(), 3000)); } return queueTimeSpan; } public ConsumeMessageService getConsumeMessageService() { return consumeMessageService; } public void setConsumeMessageService(ConsumeMessageService consumeMessageService) { this.consumeMessageService = consumeMessageService; } }