/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.rocketmq.store; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.net.SocketAddress; import java.nio.ByteBuffer; import java.nio.channels.FileLock; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicLong; import org.apache.rocketmq.common.BrokerConfig; import org.apache.rocketmq.common.MixAll; import org.apache.rocketmq.common.ServiceThread; import org.apache.rocketmq.common.SystemClock; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.UtilAll; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.common.message.MessageDecoder; import org.apache.rocketmq.common.message.MessageExt; import org.apache.rocketmq.common.message.MessageExtBatch; import org.apache.rocketmq.common.running.RunningStats; import org.apache.rocketmq.common.sysflag.MessageSysFlag; import org.apache.rocketmq.common.utils.ThreadUtils; import org.apache.rocketmq.store.config.BrokerRole; import org.apache.rocketmq.store.config.MessageStoreConfig; import org.apache.rocketmq.store.config.StorePathConfigHelper; import org.apache.rocketmq.store.ha.HAService; import org.apache.rocketmq.store.index.IndexService; import org.apache.rocketmq.store.index.QueryOffsetResult; import org.apache.rocketmq.store.schedule.ScheduleMessageService; import org.apache.rocketmq.store.stats.BrokerStatsManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; public class DefaultMessageStore implements MessageStore { private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); private static final long COMMIT_LOG_TRUNCATE_COUNT_MAX = 2; private final MessageStoreConfig messageStoreConfig; // CommitLog private final CommitLog commitLog; private final ConcurrentMap<String/* topic */, ConcurrentMap<Integer/* queueId */, ConsumeQueue>> consumeQueueTable; private final FlushConsumeQueueService flushConsumeQueueService; private final CleanCommitLogService cleanCommitLogService; private final CleanConsumeQueueService cleanConsumeQueueService; private final IndexService indexService; private final AllocateMappedFileService allocateMappedFileService; private final ReputMessageService reputMessageService; private final HAService haService; private final ScheduleMessageService scheduleMessageService; private final StoreStatsService storeStatsService; private final TransientStorePool transientStorePool; private final RunningFlags runningFlags = new RunningFlags(); private final SystemClock systemClock = new SystemClock(); private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread")); private final BrokerStatsManager brokerStatsManager; private final MessageArrivingListener messageArrivingListener; private final BrokerConfig brokerConfig; private volatile boolean shutdown = true; private StoreCheckpoint storeCheckpoint; private AtomicLong printTimes = new AtomicLong(0); private final LinkedList<CommitLogDispatcher> dispatcherList; private RandomAccessFile lockFile; private FileLock lock; boolean shutDownNormal = false; public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig) throws IOException { this.messageArrivingListener = messageArrivingListener; this.brokerConfig = brokerConfig; this.messageStoreConfig = messageStoreConfig; this.brokerStatsManager = brokerStatsManager; this.allocateMappedFileService = new AllocateMappedFileService(this); this.commitLog = new CommitLog(this); this.consumeQueueTable = new ConcurrentHashMap<>(32); this.flushConsumeQueueService = new FlushConsumeQueueService(); this.cleanCommitLogService = new CleanCommitLogService(); this.cleanConsumeQueueService = new CleanConsumeQueueService(); this.storeStatsService = new StoreStatsService(); this.indexService = new IndexService(this); this.haService = new HAService(this); this.reputMessageService = new ReputMessageService(); this.scheduleMessageService = new ScheduleMessageService(this); this.transientStorePool = new TransientStorePool(messageStoreConfig); if (messageStoreConfig.isTransientStorePoolEnable()) { this.transientStorePool.init(); } this.allocateMappedFileService.start(); this.indexService.start(); this.dispatcherList = new LinkedList<>(); this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue()); this.dispatcherList.addLast(new CommitLogDispatcherBuildIndex()); File file = new File(StorePathConfigHelper.getLockFile(messageStoreConfig.getStorePathRootDir())); MappedFile.ensureDirOK(file.getParent()); lockFile = new RandomAccessFile(file, "rw"); } public void truncateDirtyLogicFiles(long phyOffset) { ConcurrentMap<String, ConcurrentMap<Integer, ConsumeQueue>> tables = DefaultMessageStore.this.consumeQueueTable; for (ConcurrentMap<Integer, ConsumeQueue> maps : tables.values()) { for (ConsumeQueue logic : maps.values()) { logic.truncateDirtyLogicFiles(phyOffset); } } } /** * @throws IOException */ public boolean load() { boolean result = true; try { boolean lastExitOK = !this.isTempFileExist(); log.info("last shutdown {}", lastExitOK ? "normally" : "abnormally"); if (null != scheduleMessageService) { result = result && this.scheduleMessageService.load(); } // load Commit Log result = result && this.commitLog.load(); // load Consume Queue result = result && this.loadConsumeQueue(); if (result) { this.storeCheckpoint = new StoreCheckpoint(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); this.indexService.load(lastExitOK); this.recover(lastExitOK); log.info("load over, and the max phy offset = {}", this.getMaxPhyOffset()); truncateNotSync(); } } catch (Exception e) { log.error("load exception", e); result = false; } if (!result) { this.allocateMappedFileService.shutdown(); } return result; } public boolean truncateNotSync() { if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { log.warn("broker role is not slave, can not truncate data"); return true; } long phyOffsetSync = getInSyncOffsetSaved(); if (phyOffsetSync == -1) { log.info("no offset save, do not check and truncate"); return true; } long startTime = getSystemClock().now(); long phyOffsetCur = getMaxPhyOffset(); if (phyOffsetSync > phyOffsetCur) { log.warn("offset:{} > current offset:{}, no need to truncate", phyOffsetSync, phyOffsetCur); return false; } else if ((phyOffsetCur - phyOffsetSync) > COMMIT_LOG_TRUNCATE_COUNT_MAX * messageStoreConfig.getMapedFileSizeCommitLog()) { log.warn("truncate size({}) exceeded the threshold({}), do not truncate", phyOffsetCur - phyOffsetSync, COMMIT_LOG_TRUNCATE_COUNT_MAX * messageStoreConfig.getMapedFileSizeCommitLog()); return false; } indexService.deleteExpiredFile(phyOffsetSync); log.info("truncate index file to offset:{}", phyOffsetSync); //truncate commit log and consumer queue file commitLog.truncate(phyOffsetSync); log.info("truncate commit log to offset:{}", phyOffsetSync); this.recoverTopicQueueTable(); log.info("update topic queue tables"); log.info("truncate completely, from {} back to {}, cost:{} ms", phyOffsetCur, phyOffsetSync, getSystemClock().now() - startTime); return true; } private long getInSyncOffsetSaved() { long offset = -1; String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(this.messageStoreConfig.getStorePathRootDir()); try { File file = new File(fileName); if (file.exists()) { String offsetStr = MixAll.file2String(fileName); if (offsetStr != null) { offset = Long.valueOf(offsetStr); } file.delete(); } } catch (Exception ex) { log.error("get offset in sync failed", ex); } return offset; } /** * @throws Exception */ public void start() throws Exception { lock = lockFile.getChannel().tryLock(0, 1, false); if (lock == null || lock.isShared() || !lock.isValid()) { throw new RuntimeException("Lock failed,MQ already started"); } lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes())); lockFile.getChannel().force(true); this.flushConsumeQueueService.start(); this.commitLog.start(); this.storeStatsService.start(); if (this.scheduleMessageService != null && BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { this.scheduleMessageService.start(); } if (this.getMessageStoreConfig().isDuplicationEnable()) { this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset()); } else { this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset()); } this.reputMessageService.start(); this.haService.start(); this.createTempFile(); this.addScheduleTask(); if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { this.haService.initInSyncOffset(getMaxPhyOffset()); } this.shutdown = false; } public void shutdown() { if (!this.shutdown) { this.shutdown = true; this.scheduledExecutorService.shutdown(); try { Thread.sleep(1000 * 3); } catch (InterruptedException e) { log.error("shutdown Exception, ", e); } if (this.scheduleMessageService != null) { this.scheduleMessageService.shutdown(); } this.haService.shutdown(); this.storeStatsService.shutdown(); this.indexService.shutdown(); this.commitLog.shutdown(); this.reputMessageService.shutdown(); this.flushConsumeQueueService.shutdown(); this.allocateMappedFileService.shutdown(); this.storeCheckpoint.flush(); this.storeCheckpoint.shutdown(); if (this.runningFlags.isWriteable() && dispatchBehindBytes() == 0) { this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); shutDownNormal = true; } else { log.warn("the store may be wrong, so shutdown abnormally, and keep abort file."); } } this.transientStorePool.destroy(); if (lockFile != null && lock != null) { try { lock.release(); lockFile.close(); } catch (IOException e) { } } } public void destroy() { this.destroyLogics(); this.commitLog.destroy(); this.indexService.destroy(); this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); this.deleteFile(StorePathConfigHelper.getStoreCheckpoint(this.messageStoreConfig.getStorePathRootDir())); } public void destroyLogics() { for (ConcurrentMap<Integer, ConsumeQueue> maps : this.consumeQueueTable.values()) { for (ConsumeQueue logic : maps.values()) { logic.destroy(); } } } public boolean onRoleChange() { if (this.scheduleMessageService == null) { log.warn("scheduleMessageService is null"); return false; } scheduleMessageService.load(); if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) { this.scheduleMessageService.start(); this.recoverTopicQueueTable(); this.haService.initInSyncOffset(getMaxPhyOffset()); } else { this.scheduleMessageService.shutdown(); truncateNotSync(); if (this.getMessageStoreConfig().isDuplicationEnable()) { this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset()); } else { this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset()); } } log.info("role change, current role:{}", messageStoreConfig.getBrokerRole()); return true; } public PutMessageResult putMessage(MessageExtBrokerInner msg) { if (this.shutdown) { log.warn("message store has shutdown, so putMessage is forbidden"); return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); } if (BrokerRole.SLAVE == this.messageStoreConfig.getBrokerRole()) { long value = this.printTimes.getAndIncrement(); if ((value % 50000) == 0) { log.warn("message store is slave mode, so putMessage is forbidden "); } return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); } if (!this.runningFlags.isWriteable()) { long value = this.printTimes.getAndIncrement(); if ((value % 50000) == 0) { log.warn("message store is not writeable, so putMessage is forbidden " + this.runningFlags.getFlagBits()); } return new PutMessageResult(PutMessageStatus.SERVICE_NOT_AVAILABLE, null); } else { this.printTimes.set(0); } if