Java Code Examples for org.apache.rocketmq.store.config.BrokerRole

The following examples show how to use org.apache.rocketmq.store.config.BrokerRole. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: DDMQ   Source File: BrokerController.java    License: Apache License 2.0 6 votes vote down vote up
public boolean onRoleChangePre(Properties properties) {
    BrokerRole newRole = BrokerRole.valueOf(properties.get(ConfigName.BROKER_ROLE).toString());
    if (messageStoreConfig.getBrokerRole().equals(newRole) && BrokerRole.SLAVE != newRole) {
        log.warn("new role is equal to current, role:{}", messageStoreConfig.getBrokerRole());
        return false;
    }
    if (messageStoreConfig.getBrokerRole().equals(newRole) && BrokerRole.SLAVE == newRole) {
        long brokerId = Long.valueOf(properties.get(ConfigName.BROKER_ID).toString());
        if (brokerId == brokerConfig.getBrokerId()) {
            log.warn("broker id is equal, no need to change");
            return false;
        }
    }

    if (newRole.equals(BrokerRole.ASYNC_MASTER) || newRole.equals(BrokerRole.SYNC_MASTER)) {
        if (!checkNoMaster()) {
            log.error("still another master, do not init as a master");
            return false;
        }
    }

    roleChangeState = RoleChangeState.CHANGING;//will not register new role before changing completely
    log.info("role status checking pass");
    return true;
}
 
Example 2
Source Project: DDMQ   Source File: HAService.java    License: Apache License 2.0 6 votes vote down vote up
public void saveInSyncOffset() {
    if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
        return;
    }

    long minInSyncOffset = getMinOffsetInSync();
    if (minInSyncOffset == -1) {
        return;
    }

    String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(this.defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
    try {
        MixAll.string2File(String.valueOf(minInSyncOffset), fileName);
    } catch (IOException e) {
        log.error("save phy offset slave reported [{}] exception", fileName, e);
    }

    log.info("save slave min offset in sync:{}", minInSyncOffset);
}
 
Example 3
Source Project: DDMQ   Source File: HAService.java    License: Apache License 2.0 6 votes vote down vote up
public void initInSyncOffset(long offset) {
    if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
        return;
    }
    String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(this.defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
    File file = new File(fileName);
    if (file.exists()) {
        log.info("as master before, no need to sync offset");
        return;
    }

    try {
        MixAll.string2File(String.valueOf(offset), fileName);
    } catch (IOException e) {
        log.error("save phy offset slave reported [{}] exception", fileName, e);
    }

}
 
Example 4
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 6 votes vote down vote up
public boolean onRoleChange() {
    if (this.scheduleMessageService == null) {
        log.warn("scheduleMessageService is null");
        return false;
    }
    scheduleMessageService.load();
    if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        this.scheduleMessageService.start();
        this.recoverTopicQueueTable();
        this.haService.initInSyncOffset(getMaxPhyOffset());
    } else {
        this.scheduleMessageService.shutdown();
        truncateNotSync();
        if (this.getMessageStoreConfig().isDuplicationEnable()) {
            this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset());
        } else {
            this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset());
        }
    }
    log.info("role change, current role:{}", messageStoreConfig.getBrokerRole());

    return true;
}
 
Example 5
Source Project: DDMQ   Source File: DefaultMessageStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTruncate() throws Exception {
    String topic = "truncateTopic";

    for (int i = 0; i < 1000; i++) {
        MessageExtBrokerInner messageExtBrokerInner = buildMessage();
        messageExtBrokerInner.setTopic(topic);
        messageExtBrokerInner.setQueueId(0);
        messageStore.putMessage(messageExtBrokerInner);
    }
    if (messageStore instanceof DefaultMessageStore) {
        DefaultMessageStore defaultMessageStore = (DefaultMessageStore) messageStore;
        long maxPhyOffset = defaultMessageStore.getMaxPhyOffset();
        String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
        MixAll.string2File(String.valueOf(defaultMessageStore.getMaxPhyOffset() - 100), fileName);

        defaultMessageStore.getMessageStoreConfig().setBrokerRole(BrokerRole.SLAVE);
        defaultMessageStore.truncateNotSync();

        assertThat(defaultMessageStore.getMaxPhyOffset()).isEqualTo(maxPhyOffset - 100);
    }
}
 
Example 6
Source Project: DeFiBus   Source File: AdjustQueueNumStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private void adjustQueueNumByConsumerCount(String topic, AdjustType scaleType) {
    if (BrokerRole.SLAVE == this.deFiBrokerController.getMessageStoreConfig().getBrokerRole()) {
        log.info("skip adjust queue num in slave.");
        return;
    }
    if (topic.startsWith(MixAll.DLQ_GROUP_TOPIC_PREFIX) || topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX)) {
        log.info("skip adjust queue num for topic [{}]", topic);
        return;
    }
    switch (scaleType) {
        case INCREASE_QUEUE_NUM:
            adjustReadQueueNumByConsumerCount(topic, 0, scaleType);
            adjustWriteQueueNumByConsumerCount(topic, 10 * 1000, scaleType);
            break;

        case DECREASE_QUEUE_NUM:
            adjustWriteQueueNumByConsumerCount(topic, 0, scaleType);
            long delayTimeMillis = deFiBrokerController.getDeFiBusBrokerConfig().getScaleQueueSizeDelayTimeMinute() * 60 * 1000;
            adjustReadQueueNumByConsumerCount(topic, delayTimeMillis, scaleType);
            break;
    }
}
 
Example 7
Source Project: rocketmq-read   Source File: CommitLog.java    License: Apache License 2.0 6 votes vote down vote up
public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
        HAService service = this.defaultMessageStore.getHaService();
        if (messageExt.isWaitStoreMsgOK()) {
            // Determine whether to wait
            if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
                service.putRequest(request);
                service.getWaitNotifyObject().wakeupAll();
                boolean flushOK =
                    request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                if (!flushOK) {
                    log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "
                        + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());
                    putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                }
            }
            // Slave problem
            else {
                // Tell the producer, slave not available
                putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
            }
        }
    }

}
 
Example 8
Source Project: DDMQ   Source File: BrokerController.java    License: Apache License 2.0 6 votes vote down vote up
public boolean onRoleChangePre(Properties properties) {
    BrokerRole newRole = BrokerRole.valueOf(properties.get(ConfigName.BROKER_ROLE).toString());
    if (messageStoreConfig.getBrokerRole().equals(newRole) && BrokerRole.SLAVE != newRole) {
        log.warn("new role is equal to current, role:{}", messageStoreConfig.getBrokerRole());
        return false;
    }
    if (messageStoreConfig.getBrokerRole().equals(newRole) && BrokerRole.SLAVE == newRole) {
        long brokerId = Long.valueOf(properties.get(ConfigName.BROKER_ID).toString());
        if (brokerId == brokerConfig.getBrokerId()) {
            log.warn("broker id is equal, no need to change");
            return false;
        }
    }

    if (newRole.equals(BrokerRole.ASYNC_MASTER) || newRole.equals(BrokerRole.SYNC_MASTER)) {
        if (!checkNoMaster()) {
            log.error("still another master, do not init as a master");
            return false;
        }
    }

    roleChangeState = RoleChangeState.CHANGING;//will not register new role before changing completely
    log.info("role status checking pass");
    return true;
}
 
Example 9
Source Project: DDMQ   Source File: HAService.java    License: Apache License 2.0 6 votes vote down vote up
public void saveInSyncOffset() {
    if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
        return;
    }

    long minInSyncOffset = getMinOffsetInSync();
    if (minInSyncOffset == -1) {
        return;
    }

    String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(this.defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
    try {
        MixAll.string2File(String.valueOf(minInSyncOffset), fileName);
    } catch (IOException e) {
        log.error("save phy offset slave reported [{}] exception", fileName, e);
    }

    log.info("save slave min offset in sync:{}", minInSyncOffset);
}
 
Example 10
Source Project: DDMQ   Source File: HAService.java    License: Apache License 2.0 6 votes vote down vote up
public void initInSyncOffset(long offset) {
    if (this.defaultMessageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) {
        return;
    }
    String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(this.defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
    File file = new File(fileName);
    if (file.exists()) {
        log.info("as master before, no need to sync offset");
        return;
    }

    try {
        MixAll.string2File(String.valueOf(offset), fileName);
    } catch (IOException e) {
        log.error("save phy offset slave reported [{}] exception", fileName, e);
    }

}
 
Example 11
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 6 votes vote down vote up
public boolean onRoleChange() {
    if (this.scheduleMessageService == null) {
        log.warn("scheduleMessageService is null");
        return false;
    }
    scheduleMessageService.load();
    if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        this.scheduleMessageService.start();
        this.recoverTopicQueueTable();
        this.haService.initInSyncOffset(getMaxPhyOffset());
    } else {
        this.scheduleMessageService.shutdown();
        truncateNotSync();
        if (this.getMessageStoreConfig().isDuplicationEnable()) {
            this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset());
        } else {
            this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset());
        }
    }
    log.info("role change, current role:{}", messageStoreConfig.getBrokerRole());

    return true;
}
 
Example 12
Source Project: DDMQ   Source File: DefaultMessageStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTruncate() throws Exception {
    String topic = "truncateTopic";

    for (int i = 0; i < 1000; i++) {
        MessageExtBrokerInner messageExtBrokerInner = buildMessage();
        messageExtBrokerInner.setTopic(topic);
        messageExtBrokerInner.setQueueId(0);
        messageStore.putMessage(messageExtBrokerInner);
    }
    if (messageStore instanceof DefaultMessageStore) {
        DefaultMessageStore defaultMessageStore = (DefaultMessageStore) messageStore;
        long maxPhyOffset = defaultMessageStore.getMaxPhyOffset();
        String fileName = StorePathConfigHelper.getOffsetInSyncStorePath(defaultMessageStore.getMessageStoreConfig().getStorePathRootDir());
        MixAll.string2File(String.valueOf(defaultMessageStore.getMaxPhyOffset() - 100), fileName);

        defaultMessageStore.getMessageStoreConfig().setBrokerRole(BrokerRole.SLAVE);
        defaultMessageStore.truncateNotSync();

        assertThat(defaultMessageStore.getMaxPhyOffset()).isEqualTo(maxPhyOffset - 100);
    }
}
 
Example 13
Source Project: rocketmq   Source File: BrokerController.java    License: Apache License 2.0 6 votes vote down vote up
private void handleSlaveSynchronize(BrokerRole role) {
    if (role == BrokerRole.SLAVE) {
        if (null != slaveSyncFuture) {
            slaveSyncFuture.cancel(false);
        }
        this.slaveSynchronize.setMasterAddr(null);
        slaveSyncFuture = this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                try {
                    BrokerController.this.slaveSynchronize.syncAll();
                }
                catch (Throwable e) {
                    log.error("ScheduledTask SlaveSynchronize syncAll error.", e);
                }
            }
        }, 1000 * 3, 1000 * 10, TimeUnit.MILLISECONDS);
    } else {
        //handle the slave synchronise
        if (null != slaveSyncFuture) {
            slaveSyncFuture.cancel(false);
        }
        this.slaveSynchronize.setMasterAddr(null);
    }
}
 
Example 14
Source Project: rocketmq   Source File: CommitLog.java    License: Apache License 2.0 6 votes vote down vote up
public CompletableFuture<PutMessageStatus> submitReplicaRequest(AppendMessageResult result, PutMessageResult putMessageResult,
                                                    MessageExt messageExt) {
    if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
        HAService service = this.defaultMessageStore.getHaService();
        if (messageExt.isWaitStoreMsgOK()) {
            if (service.isSlaveOK(result.getWroteBytes() + result.getWroteOffset())) {
                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(),
                        this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                service.putRequest(request);
                service.getWaitNotifyObject().wakeupAll();
                return request.future();
            }
            else {
                return CompletableFuture.completedFuture(PutMessageStatus.SLAVE_NOT_AVAILABLE);
            }
        }
    }
    return CompletableFuture.completedFuture(PutMessageStatus.PUT_OK);
}
 
Example 15
Source Project: rocketmq   Source File: HATest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void init() throws Exception {
    StoreHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123);
    BornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0);
    masterMessageStoreConfig = new MessageStoreConfig();
    masterMessageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER);
    masterMessageStoreConfig.setStorePathRootDir(storePathRootDir+File.separator+"master");
    masterMessageStoreConfig.setStorePathCommitLog(storePathRootDir+File.separator+"master"+ File.separator+"commitlog");
    buildMessageStoreConfig(masterMessageStoreConfig);
    slaveStoreConfig = new MessageStoreConfig();
    slaveStoreConfig.setBrokerRole(BrokerRole.SLAVE);
    slaveStoreConfig.setStorePathRootDir(storePathRootDir+File.separator+"slave");
    slaveStoreConfig.setStorePathCommitLog(storePathRootDir+File.separator+"slave"+ File.separator+"commitlog");
    slaveStoreConfig.setHaListenPort(10943);
    buildMessageStoreConfig(slaveStoreConfig);
    messageStore = buildMessageStore(masterMessageStoreConfig,0L);
    slaveMessageStore = buildMessageStore(slaveStoreConfig,1L);
    boolean load = messageStore.load();
    boolean slaveLoad = slaveMessageStore.load();
    slaveMessageStore.updateHaMasterAddress("127.0.0.1:10912");
    assertTrue(load);
    assertTrue(slaveLoad);
    messageStore.start();
    slaveMessageStore.start();
    Thread.sleep(6000L);//because the haClient will wait 5s after the first connectMaster failed,sleep 6s
}
 
Example 16
Source Project: DDMQ   Source File: CommitLog.java    License: Apache License 2.0 5 votes vote down vote up
private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) {
    ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();

    int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSTION);
    if (magicCode != MESSAGE_MAGIC_CODE) {
        return false;
    }

    long storeTimestamp = byteBuffer.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION);
    if (0 == storeTimestamp) {
        return false;
    }

    if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()
        && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {
        if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
            log.info("find check timestamp, {} {}",
                storeTimestamp,
                UtilAll.timeMillisToHumanString(storeTimestamp));
            return true;
        }
    } else {
        long minTimestamp = this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp();
        if (BrokerRole.SLAVE == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
            minTimestamp = this.defaultMessageStore.getStoreCheckpoint().getLogicsMsgTimestamp();
        }
        if (storeTimestamp <= minTimestamp) {
            log.info("find check timestamp, {} {}",
                storeTimestamp,
                UtilAll.timeMillisToHumanString(storeTimestamp));
            return true;
        }
    }

    return false;
}
 
Example 17
Source Project: DDMQ   Source File: CommitLog.java    License: Apache License 2.0 5 votes vote down vote up
public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
        HAService service = this.defaultMessageStore.getHaService();
        if (messageExt.isWaitStoreMsgOK()) {
            // Determine whether to wait
            if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(),
                    this.defaultMessageStore.getSystemClock().now() + this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                service.putRequest(request);
                service.getWaitNotifyObject().wakeupAll();
                boolean flushOK =
                    request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                if (!flushOK) {
                    log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "
                        + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());
                    putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                }
            }
            // Slave problem
            else {
                // Tell the producer, slave not available
                putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
            }
        }
    }

}
 
Example 18
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
public boolean truncateNotSync() {
    if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        log.warn("broker role is not slave, can not truncate data");
        return true;
    }

    long phyOffsetSync = getInSyncOffsetSaved();
    if (phyOffsetSync == -1) {
        log.info("no offset save, do not check and truncate");
        return true;
    }

    long startTime = getSystemClock().now();
    long phyOffsetCur = getMaxPhyOffset();
    if (phyOffsetSync > phyOffsetCur) {
        log.warn("offset:{} > current offset:{}, no need to truncate", phyOffsetSync, phyOffsetCur);
        return false;
    } else if ((phyOffsetCur - phyOffsetSync) > COMMIT_LOG_TRUNCATE_COUNT_MAX * messageStoreConfig.getMapedFileSizeCommitLog()) {
        log.warn("truncate size({}) exceeded the threshold({}), do not truncate", phyOffsetCur - phyOffsetSync,
            COMMIT_LOG_TRUNCATE_COUNT_MAX * messageStoreConfig.getMapedFileSizeCommitLog());
        return false;
    }

    indexService.deleteExpiredFile(phyOffsetSync);
    log.info("truncate index file to offset:{}", phyOffsetSync);

    //truncate commit log and consumer queue file
    commitLog.truncate(phyOffsetSync);
    log.info("truncate commit log to offset:{}", phyOffsetSync);

    this.recoverTopicQueueTable();
    log.info("update topic queue tables");

    log.info("truncate completely, from {} back to {}, cost:{} ms", phyOffsetCur, phyOffsetSync, getSystemClock().now() - startTime);

    return true;
}
 
Example 19
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @throws Exception
 */
public void start() throws Exception {

    lock = lockFile.getChannel().tryLock(0, 1, false);
    if (lock == null || lock.isShared() || !lock.isValid()) {
        throw new RuntimeException("Lock failed,MQ already started");
    }

    lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes()));
    lockFile.getChannel().force(true);

    this.flushConsumeQueueService.start();
    this.commitLog.start();
    this.storeStatsService.start();

    if (this.scheduleMessageService != null && BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        this.scheduleMessageService.start();
    }

    if (this.getMessageStoreConfig().isDuplicationEnable()) {
        this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset());
    } else {
        this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset());
    }
    this.reputMessageService.start();

    this.haService.start();

    this.createTempFile();
    this.addScheduleTask();
    if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        this.haService.initInSyncOffset(getMaxPhyOffset());
    }
    this.shutdown = false;
}
 
Example 20
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
private long nextOffsetCorrection(long oldOffset, long newOffset) {
    long nextOffset = oldOffset;
    if (this.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE || this.getMessageStoreConfig().isOffsetCheckInSlave()) {
        nextOffset = newOffset;
    }
    return nextOffset;
}
 
Example 21
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
private boolean isBuildIndex() {
    if (DefaultMessageStore.this.messageStoreConfig.isMessageIndexEnable()) {
        if (DefaultMessageStore.this.messageStoreConfig.isMessageIndexOnlySlaveEnable()) {
            return BrokerRole.SLAVE.equals(DefaultMessageStore.this.messageStoreConfig.getBrokerRole());
        } else {
            return true;
        }
    }
    return false;
}
 
Example 22
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
private void dispatchAndNotify(DispatchRequest dispatchRequest) {
    DefaultMessageStore.this.doDispatch(dispatchRequest);
    if (BrokerRole.SLAVE != DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole()
        && DefaultMessageStore.this.brokerConfig.isLongPollingEnable()) {
        DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(),
            dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1,
            dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(),
            dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap());
    }
}
 
Example 23
Source Project: rocketmq-4.3.0   Source File: CommitLog.java    License: Apache License 2.0 5 votes vote down vote up
public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
//        如果master同步刷新
        if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
            HAService service = this.defaultMessageStore.getHaService();
            if (messageExt.isWaitStoreMsgOK()) {
                // Determine whether to wait
                if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
                    GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes());
                    service.putRequest(request);
                    service.getWaitNotifyObject().wakeupAll();
//                    countDownLatch.await 同步等待刷新,除非等待超时
                    boolean flushOK =
                        request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                    if (!flushOK) {
                        log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "
                            + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());
                        putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                    }
                }
                // Slave problem
                else {
                    // Tell the producer, slave not available
                    putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
                }
            }
        }

    }
 
Example 24
Source Project: rocketmq-4.3.0   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
private long nextOffsetCorrection(long oldOffset, long newOffset) {
        long nextOffset = oldOffset;
//        broker不是slave,检查在slave的offset
        if (this.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE || this.getMessageStoreConfig().isOffsetCheckInSlave()) {
            nextOffset = newOffset;
        }
        return nextOffset;
    }
 
Example 25
Source Project: rocketmq-read   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 获取下一个偏移量
 */
private long nextOffsetCorrection(long oldOffset, long newOffset) {
    long nextOffset = oldOffset;
    //如果是master或者从服务器坚持 offset检测。置为newOffset
    if (this.getMessageStoreConfig().getBrokerRole() != BrokerRole.SLAVE || this.getMessageStoreConfig().isOffsetCheckInSlave()) {
        nextOffset = newOffset;
    }
    return nextOffset;
}
 
Example 26
Source Project: rocketmq   Source File: BrokerControllerSlaveTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
     * broker 启动
     */
    @Test
    public void testBrokerRestart() throws Exception {
        System.setProperty("user.home", System.getProperty("user.home") + File.separator + "broker" + File.separator + "slave");

        // 设置版本号
        System.setProperty(RemotingCommand.REMOTING_VERSION_KEY, Integer.toString(MQVersion.CURRENT_VERSION));
        //
        final NettyServerConfig nettyServerConfig = new NettyServerConfig();
        nettyServerConfig.setListenPort(20911);
        //
        final BrokerConfig brokerConfig = new BrokerConfig();
        brokerConfig.setBrokerName("broker-a");
        brokerConfig.setBrokerId(1L); // 1
        brokerConfig.setNamesrvAddr("127.0.0.1:9876");

        //
        final MessageStoreConfig messageStoreConfig = new MessageStoreConfig();
        messageStoreConfig.setDeleteWhen("04");
        messageStoreConfig.setFileReservedTime(48);
        messageStoreConfig.setFlushDiskType(FlushDiskType.ASYNC_FLUSH);
        messageStoreConfig.setBrokerRole(BrokerRole.SLAVE);

        messageStoreConfig.setDuplicationEnable(false);

        messageStoreConfig.setHaListenPort(nettyServerConfig.getListenPort() + 1);

//        BrokerPathConfigHelper.setBrokerConfigPath("/Users/yunai/百度云同步盘/开发/Javascript/Story/incubator-rocketmq/conf/broker.conf");
        // broker 启动
        BrokerController brokerController = new BrokerController(//
                brokerConfig, //
                nettyServerConfig, //
                new NettyClientConfig(), //
                messageStoreConfig);
        brokerController.initialize();
        brokerController.start();
//        brokerController.getTopicConfigManager().createTopicInSendMessageBackMethod()
        Thread.sleep(DateUtils.MILLIS_PER_DAY);
    }
 
Example 27
Source Project: DDMQ   Source File: CommitLog.java    License: Apache License 2.0 5 votes vote down vote up
private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) {
    ByteBuffer byteBuffer = mappedFile.sliceByteBuffer();

    int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSTION);
    if (magicCode != MESSAGE_MAGIC_CODE) {
        return false;
    }

    long storeTimestamp = byteBuffer.getLong(MessageDecoder.MESSAGE_STORE_TIMESTAMP_POSTION);
    if (0 == storeTimestamp) {
        return false;
    }

    if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable()
        && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) {
        if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) {
            log.info("find check timestamp, {} {}",
                storeTimestamp,
                UtilAll.timeMillisToHumanString(storeTimestamp));
            return true;
        }
    } else {
        long minTimestamp = this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp();
        if (BrokerRole.SLAVE == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
            minTimestamp = this.defaultMessageStore.getStoreCheckpoint().getLogicsMsgTimestamp();
        }
        if (storeTimestamp <= minTimestamp) {
            log.info("find check timestamp, {} {}",
                storeTimestamp,
                UtilAll.timeMillisToHumanString(storeTimestamp));
            return true;
        }
    }

    return false;
}
 
Example 28
Source Project: DDMQ   Source File: CommitLog.java    License: Apache License 2.0 5 votes vote down vote up
public void handleHA(AppendMessageResult result, PutMessageResult putMessageResult, MessageExt messageExt) {
    if (BrokerRole.SYNC_MASTER == this.defaultMessageStore.getMessageStoreConfig().getBrokerRole()) {
        HAService service = this.defaultMessageStore.getHaService();
        if (messageExt.isWaitStoreMsgOK()) {
            // Determine whether to wait
            if (service.isSlaveOK(result.getWroteOffset() + result.getWroteBytes())) {
                GroupCommitRequest request = new GroupCommitRequest(result.getWroteOffset() + result.getWroteBytes(),
                    this.defaultMessageStore.getSystemClock().now() + this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                service.putRequest(request);
                service.getWaitNotifyObject().wakeupAll();
                boolean flushOK =
                    request.waitForFlush(this.defaultMessageStore.getMessageStoreConfig().getSyncFlushTimeout());
                if (!flushOK) {
                    log.error("do sync transfer other node, wait return, but failed, topic: " + messageExt.getTopic() + " tags: "
                        + messageExt.getTags() + " client address: " + messageExt.getBornHostNameString());
                    putMessageResult.setPutMessageStatus(PutMessageStatus.FLUSH_SLAVE_TIMEOUT);
                }
            }
            // Slave problem
            else {
                // Tell the producer, slave not available
                putMessageResult.setPutMessageStatus(PutMessageStatus.SLAVE_NOT_AVAILABLE);
            }
        }
    }

}
 
Example 29
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
public boolean truncateNotSync() {
    if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        log.warn("broker role is not slave, can not truncate data");
        return true;
    }

    long phyOffsetSync = getInSyncOffsetSaved();
    if (phyOffsetSync == -1) {
        log.info("no offset save, do not check and truncate");
        return true;
    }

    long startTime = getSystemClock().now();
    long phyOffsetCur = getMaxPhyOffset();
    if (phyOffsetSync > phyOffsetCur) {
        log.warn("offset:{} > current offset:{}, no need to truncate", phyOffsetSync, phyOffsetCur);
        return false;
    } else if ((phyOffsetCur - phyOffsetSync) > COMMIT_LOG_TRUNCATE_COUNT_MAX * messageStoreConfig.getMapedFileSizeCommitLog()) {
        log.warn("truncate size({}) exceeded the threshold({}), do not truncate", phyOffsetCur - phyOffsetSync,
            COMMIT_LOG_TRUNCATE_COUNT_MAX * messageStoreConfig.getMapedFileSizeCommitLog());
        return false;
    }

    indexService.deleteExpiredFile(phyOffsetSync);
    log.info("truncate index file to offset:{}", phyOffsetSync);

    //truncate commit log and consumer queue file
    commitLog.truncate(phyOffsetSync);
    log.info("truncate commit log to offset:{}", phyOffsetSync);

    this.recoverTopicQueueTable();
    log.info("update topic queue tables");

    log.info("truncate completely, from {} back to {}, cost:{} ms", phyOffsetCur, phyOffsetSync, getSystemClock().now() - startTime);

    return true;
}
 
Example 30
Source Project: DDMQ   Source File: DefaultMessageStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @throws Exception
 */
public void start() throws Exception {

    lock = lockFile.getChannel().tryLock(0, 1, false);
    if (lock == null || lock.isShared() || !lock.isValid()) {
        throw new RuntimeException("Lock failed,MQ already started");
    }

    lockFile.getChannel().write(ByteBuffer.wrap("lock".getBytes()));
    lockFile.getChannel().force(true);

    this.flushConsumeQueueService.start();
    this.commitLog.start();
    this.storeStatsService.start();

    if (this.scheduleMessageService != null && BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        this.scheduleMessageService.start();
    }

    if (this.getMessageStoreConfig().isDuplicationEnable()) {
        this.reputMessageService.setReputFromOffset(this.commitLog.getConfirmOffset());
    } else {
        this.reputMessageService.setReputFromOffset(this.commitLog.getMaxOffset());
    }
    this.reputMessageService.start();

    this.haService.start();

    this.createTempFile();
    this.addScheduleTask();
    if (BrokerRole.SLAVE != messageStoreConfig.getBrokerRole()) {
        this.haService.initInSyncOffset(getMaxPhyOffset());
    }
    this.shutdown = false;
}