Java Code Examples for org.apache.rocketmq.common.protocol.heartbeat.MessageModel

The following examples show how to use org.apache.rocketmq.common.protocol.heartbeat.MessageModel. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: netty-chat   Source File: BroadcastConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("example_group_name");

    consumer.setNamesrvAddr("127.0.0.1:9876");
    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

    //set to broadcast mode
    consumer.setMessageModel(MessageModel.BROADCASTING);

    consumer.subscribe("TopicTest", "TagA || TagC || TagD");

    consumer.registerMessageListener(new MessageListenerConcurrently() {

        @Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
                                                        ConsumeConcurrentlyContext context) {
            System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n");
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        }
    });

    consumer.start();
    System.out.printf("Broadcast Consumer Started.%n");
}
 
Example 2
Source Project: rocketmq-read   Source File: PushConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws InterruptedException, MQClientException {
    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1");

    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

    consumer.setMessageModel(MessageModel.BROADCASTING);

    consumer.subscribe("TopicTest", "TagA || TagC || TagD");

    consumer.registerMessageListener(new MessageListenerConcurrently() {

        @Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
            ConsumeConcurrentlyContext context) {
            System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        }
    });

    consumer.start();
    System.out.printf("Broadcast Consumer Started.%n");
}
 
Example 3
Source Project: DDMQ   Source File: ConsumerManager.java    License: Apache License 2.0 6 votes vote down vote up
public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
    ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
    final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {

    ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
    if (null == consumerGroupInfo) {
        ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
        ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
        consumerGroupInfo = prev != null ? prev : tmp;
    }

    boolean r1 =
        consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel,
            consumeFromWhere);
    boolean r2 = consumerGroupInfo.updateSubscription(subList);

    if (r1 || r2) {
        if (isNotifyConsumerIdsChangedEnable) {
            this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
        }
    }

    this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);

    return r1 || r2;
}
 
Example 4
Source Project: DDMQ   Source File: RebalancePushImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMessageQueueChanged_SizeThreshold() {
    RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,
        new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);
    init(rebalancePush);

    // Just set pullThresholdSizeForQueue
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(1024);
    Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 0));
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 1));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(1024);

    // Set pullThresholdSizeForTopic
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForTopic(1024);
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(512);

    // Change message queue allocate result
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 2));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(341);
}
 
Example 5
Source Project: DeFiBus   Source File: AdjustQueueNumStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private int adjustQueueSizeByMaxConsumerCount(String topic) {
    int queueSize = this.deFiBrokerController.getDeFiBusBrokerConfig().getMinQueueNum();
    Set<String> maxCidList = null;
    Set<String> topicConsumeByWho = this.deFiBrokerController.getConsumerManager().queryTopicConsumeByWho(topic);
    for (String group : topicConsumeByWho) {
        DeFiConsumerGroupInfo consumerGroupInfo = (DeFiConsumerGroupInfo) this.deFiBrokerController.getConsumerManager().getConsumerGroupInfo(group);
        if (consumerGroupInfo != null && consumerGroupInfo.getMessageModel() == MessageModel.CLUSTERING) {
            Set<String> cidList = consumerGroupInfo.getClientIdBySubscription(topic);
            if (cidList != null) {
                int scaleSize = this.scaleQueueSize(cidList);
                if (scaleSize >= queueSize) {
                    queueSize = scaleSize;
                    maxCidList = cidList;
                }
            }
        }
    }
    log.info("calculate queue size by max consumer count, result: {} cidList: {}", queueSize, maxCidList);
    return queueSize;
}
 
Example 6
@Override
public void messageQueueChanged(String topic, Set<MessageQueue> mqAll, Set<MessageQueue> mqDivided) {
    MessageModel messageModel =
        MQPullConsumerScheduleService.this.defaultMQPullConsumer.getMessageModel();
    switch (messageModel) {
        //广播消费
        case BROADCASTING:
            MQPullConsumerScheduleService.this.putTask(topic, mqAll);
            break;
            //集群消费
        case CLUSTERING:
            MQPullConsumerScheduleService.this.putTask(topic, mqDivided);
            break;
        default:
            break;
    }
}
 
Example 7
Source Project: DDMQ   Source File: RebalancePushImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMessageQueueChanged_CountThreshold() {
    RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,
        new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);
    init(rebalancePush);

    // Just set pullThresholdForQueue
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForQueue(1024);
    Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 0));
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 1));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(1024);

    // Set pullThresholdForTopic
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdForTopic(1024);
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(512);

    // Change message queue allocate result
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 2));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdForQueue()).isEqualTo(341);
}
 
Example 8
Source Project: rocketmq   Source File: PushConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws InterruptedException, MQClientException {
        DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1");

        consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

        consumer.setMessageModel(MessageModel.BROADCASTING);

//        consumer.subscribe("TopicTest", "TagA || TagC || TagD");
        consumer.subscribe("TopicTest", "*");

        consumer.setNamesrvAddr("127.0.0.1:9876"); // TODO add by yunai

        consumer.registerMessageListener(new MessageListenerConcurrently() {

            @Override
            public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
                ConsumeConcurrentlyContext context) {
                System.out.printf(Thread.currentThread().getName() + " Receive New Messages: " + msgs + "%n");
                return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
            }
        });

        consumer.start();
        System.out.printf("Broadcast Consumer Started.%n");
    }
 
Example 9
Source Project: DDMQ   Source File: ConsumerConnectionSubCommandTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);

    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);

    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    connections.add(new Connection());
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
}
 
Example 10
@BeforeClass
public static void init() throws NoSuchFieldException, IllegalAccessException, InterruptedException, RemotingTimeoutException, MQClientException, RemotingSendRequestException, RemotingConnectException, MQBrokerException {
    mQClientAPIImpl = mock(MQClientAPIImpl.class);
    defaultMQAdminExt = new DefaultMQAdminExt();
    defaultMQAdminExtImpl = new DefaultMQAdminExtImpl(defaultMQAdminExt, 1000);

    Field field = DefaultMQAdminExtImpl.class.getDeclaredField("mqClientInstance");
    field.setAccessible(true);
    field.set(defaultMQAdminExtImpl, mqClientInstance);
    field = MQClientInstance.class.getDeclaredField("mQClientAPIImpl");
    field.setAccessible(true);
    field.set(mqClientInstance, mQClientAPIImpl);
    field = DefaultMQAdminExt.class.getDeclaredField("defaultMQAdminExtImpl");
    field.setAccessible(true);
    field.set(defaultMQAdminExt, defaultMQAdminExtImpl);

    ConsumerConnection consumerConnection = new ConsumerConnection();
    consumerConnection.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerConnection.setMessageModel(MessageModel.CLUSTERING);
    HashSet<Connection> connections = new HashSet<>();
    connections.add(new Connection());
    consumerConnection.setConnectionSet(connections);
    consumerConnection.setSubscriptionTable(new ConcurrentHashMap<String, SubscriptionData>());
    consumerConnection.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    when(mQClientAPIImpl.getConsumerConnectionList(anyString(), anyString(), anyLong())).thenReturn(consumerConnection);
}
 
Example 11
Source Project: rocketmq-4.3.0   Source File: PushConsumer.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws InterruptedException, MQClientException {
    DefaultMQPushConsumer consumer = new DefaultMQPushConsumer("please_rename_unique_group_name_1");

    consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);

    consumer.setMessageModel(MessageModel.BROADCASTING);

    consumer.subscribe("TopicTest", "TagA || TagC || TagD");

    consumer.registerMessageListener(new MessageListenerConcurrently() {

        @Override
        public ConsumeConcurrentlyStatus consumeMessage(List<MessageExt> msgs,
            ConsumeConcurrentlyContext context) {
            System.out.printf("%s Receive New Messages: %s %n", Thread.currentThread().getName(), msgs);
            return ConsumeConcurrentlyStatus.CONSUME_SUCCESS;
        }
    });

    consumer.start();
    System.out.printf("Broadcast Consumer Started.%n");
}
 
Example 12
Source Project: DeFiBus   Source File: ResponseConsumerAutoReply.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    DeFiBusClientConfig deFiBusClientConfig = new DeFiBusClientConfig();
    deFiBusClientConfig.setNamesrvAddr("");
    DeFiBusProducer deFiBusProducer = new DeFiBusProducer(deFiBusClientConfig);
    DeFiBusPushConsumer deFiBusPushConsumer = new DeFiBusPushConsumer(deFiBusClientConfig);
    deFiBusPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET);
    deFiBusPushConsumer.getDefaultMQPushConsumer().setMessageModel(MessageModel.CLUSTERING);
    deFiBusPushConsumer.registerMessageListener(new DeFiBusMessageListenerConcurrentlyWithReply(deFiBusProducer) {
        @Override
        public String handleMessage(MessageExt msg, ConsumeConcurrentlyContext context) {
            //1. biz handle logic

            //2. create reply content
            String replyContent = "A reply message content";

            return replyContent;
        }
    });
    try {
        deFiBusProducer.start();
        deFiBusPushConsumer.subscribe("REQUEST_REPLY_TOPIC");
        deFiBusPushConsumer.start();
    } catch (MQClientException e) {
        e.printStackTrace();
    }
}
 
Example 13
Source Project: rocketmq-read   Source File: ConsumerManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 注册一个消费者
 * @param group group
 * @param clientChannelInfo clientChannelInfo
 * @param consumeType consumeType
 * @param messageModel 消息模式
 * @param consumeFromWhere consumerFromWhere
 * @param subList  SubscriptionData
 * @param isNotifyConsumerIdsChangedEnable isNotifyConsumerIdsChangedEnable
 * @return ;
 */
public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
    ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
    final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {

    ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
    if (null == consumerGroupInfo) {
        ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
        ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
        consumerGroupInfo = prev != null ? prev : tmp;
    }

    boolean r1 = consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel, consumeFromWhere);
    boolean r2 = consumerGroupInfo.updateSubscription(subList);

    if (r1 || r2) {
        if (isNotifyConsumerIdsChangedEnable) {
            this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
        }
    }

    this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);

    return r1 || r2;
}
 
Example 14
Source Project: rocketmq-4.3.0   Source File: RebalancePushImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMessageQueueChanged_SizeThreshold() {
    RebalancePushImpl rebalancePush = new RebalancePushImpl(consumerGroup, MessageModel.CLUSTERING,
        new AllocateMessageQueueAveragely(), mqClientInstance, defaultMQPushConsumer);
    init(rebalancePush);

    // Just set pullThresholdSizeForQueue
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForQueue(1024);
    Set<MessageQueue> allocateResultSet = new HashSet<MessageQueue>();
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 0));
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 1));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(1024);

    // Set pullThresholdSizeForTopic
    defaultMQPushConsumer.getDefaultMQPushConsumer().setPullThresholdSizeForTopic(1024);
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(512);

    // Change message queue allocate result
    allocateResultSet.add(new MessageQueue(topic, "BrokerA", 2));
    doRebalanceForcibly(rebalancePush, allocateResultSet);
    assertThat(defaultMQPushConsumer.getDefaultMQPushConsumer().getPullThresholdSizeForQueue()).isEqualTo(341);
}
 
Example 15
@PostConstruct
public void init() {
    defaultMQPushConsumer =
            new DefaultMQPushConsumer(
                MessageProtocolConst.SECKILL_CHARGE_ORDER_TOPIC.getConsumerGroup(),
                    new AclClientRPCHook(new SessionCredentials(aclAccessKey, aclAccessSecret)),
                    // 平均分配队列算法,hash
                    new AllocateMessageQueueAveragely());
    defaultMQPushConsumer.setNamesrvAddr(namesrvConfig.nameSrvAddr());
    // 从头开始消费
    defaultMQPushConsumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    // 消费模式:集群模式
    // 集群:同一条消息 只会被一个消费者节点消费到
    // 广播:同一条消息 每个消费者都会消费到
    defaultMQPushConsumer.setMessageModel(MessageModel.CLUSTERING);
    // 注册监听器
    defaultMQPushConsumer.registerMessageListener(messageListener);
    // 设置每次拉取的消息量,默认为1
    defaultMQPushConsumer.setConsumeMessageBatchMaxSize(1);
    // 订阅所有消息
    try {
        defaultMQPushConsumer.subscribe(MessageProtocolConst.SECKILL_CHARGE_ORDER_TOPIC.getTopic(), "*");
        // 启动消费者
        defaultMQPushConsumer.start();
    } catch (MQClientException e) {
        LOGGER.error("[秒杀下单消费者]--SecKillChargeOrderConsumer加载异常!e={}", LogExceptionWapper.getStackTrace(e));
        throw new RuntimeException("[秒杀下单消费者]--SecKillChargeOrderConsumer加载异常!", e);
    }
    LOGGER.info("[秒杀下单消费者]--SecKillChargeOrderConsumer加载完成!");
}
 
Example 16
Source Project: rocketmq   Source File: MQHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static void resetOffsetByTimestamp(
    final MessageModel messageModel,
    final String consumerGroup,
    final String topic,
    final long timestamp) throws Exception {
    resetOffsetByTimestamp(messageModel, "DEFAULT", consumerGroup, topic, timestamp);
}
 
Example 17
Source Project: rocketmq   Source File: MQHelper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reset consumer topic offset according to time
 *
 * @param messageModel which model
 * @param instanceName which instance
 * @param consumerGroup consumer group
 * @param topic topic
 * @param timestamp time
 * @throws Exception
 */
public static void resetOffsetByTimestamp(
    final MessageModel messageModel,
    final String instanceName,
    final String consumerGroup,
    final String topic,
    final long timestamp) throws Exception {
    final Logger log = ClientLogger.getLog();

    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setInstanceName(instanceName);
    consumer.setMessageModel(messageModel);
    consumer.start();

    Set<MessageQueue> mqs = null;
    try {
        mqs = consumer.fetchSubscribeMessageQueues(topic);
        if (mqs != null && !mqs.isEmpty()) {
            TreeSet<MessageQueue> mqsNew = new TreeSet<MessageQueue>(mqs);
            for (MessageQueue mq : mqsNew) {
                long offset = consumer.searchOffset(mq, timestamp);
                if (offset >= 0) {
                    consumer.updateConsumeOffset(mq, offset);
                    log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}",
                        consumerGroup, offset, mq);
                }
            }
        }
    } catch (Exception e) {
        log.warn("resetOffsetByTimestamp Exception", e);
        throw e;
    } finally {
        if (mqs != null) {
            consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs);
        }
        consumer.shutdown();
    }
}
 
Example 18
Source Project: rocketmq-4.3.0   Source File: ConsumerManager.java    License: Apache License 2.0 5 votes vote down vote up
public boolean registerConsumer(final String group, final ClientChannelInfo clientChannelInfo,
        ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere,
        final Set<SubscriptionData> subList, boolean isNotifyConsumerIdsChangedEnable) {

//        获取消费组信息
        ConsumerGroupInfo consumerGroupInfo = this.consumerTable.get(group);
        if (null == consumerGroupInfo) {
            ConsumerGroupInfo tmp = new ConsumerGroupInfo(group, consumeType, messageModel, consumeFromWhere);
            ConsumerGroupInfo prev = this.consumerTable.putIfAbsent(group, tmp);
            consumerGroupInfo = prev != null ? prev : tmp;
        }

//        更新channel信息=》
        boolean r1 =
            consumerGroupInfo.updateChannel(clientChannelInfo, consumeType, messageModel,
                consumeFromWhere);
//       更新订阅细信息=》
        boolean r2 = consumerGroupInfo.updateSubscription(subList);

        if (r1 || r2) {
            if (isNotifyConsumerIdsChangedEnable) {
//                通知消费组的所有消费者channel和订阅信息改变了=》
                this.consumerIdsChangeListener.handle(ConsumerGroupEvent.CHANGE, group, consumerGroupInfo.getAllChannel());
            }
        }

        this.consumerIdsChangeListener.handle(ConsumerGroupEvent.REGISTER, group, subList);

        return r1 || r2;
    }
 
Example 19
Source Project: rocketmq-read   Source File: RebalancePullImpl.java    License: Apache License 2.0 5 votes vote down vote up
public RebalancePullImpl(String consumerGroup, MessageModel messageModel,
    AllocateMessageQueueStrategy allocateMessageQueueStrategy,
    MQClientInstance mQClientFactory, DefaultMQPullConsumerImpl defaultMQPullConsumerImpl) {

    super(consumerGroup, messageModel, allocateMessageQueueStrategy, mQClientFactory);
    this.defaultMQPullConsumerImpl = defaultMQPullConsumerImpl;
}
 
Example 20
Source Project: DDMQ   Source File: MQHelper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reset consumer topic offset according to time
 *
 * @param messageModel which model
 * @param instanceName which instance
 * @param consumerGroup consumer group
 * @param topic topic
 * @param timestamp time
 */
public static void resetOffsetByTimestamp(
    final MessageModel messageModel,
    final String instanceName,
    final String consumerGroup,
    final String topic,
    final long timestamp) throws Exception {
    final Logger log = ClientLogger.getLog();

    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setInstanceName(instanceName);
    consumer.setMessageModel(messageModel);
    consumer.start();

    Set<MessageQueue> mqs = null;
    try {
        mqs = consumer.fetchSubscribeMessageQueues(topic);
        if (mqs != null && !mqs.isEmpty()) {
            TreeSet<MessageQueue> mqsNew = new TreeSet<MessageQueue>(mqs);
            for (MessageQueue mq : mqsNew) {
                long offset = consumer.searchOffset(mq, timestamp);
                if (offset >= 0) {
                    consumer.updateConsumeOffset(mq, offset);
                    log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}",
                        consumerGroup, offset, mq);
                }
            }
        }
    } catch (Exception e) {
        log.warn("resetOffsetByTimestamp Exception", e);
        throw e;
    } finally {
        if (mqs != null) {
            consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs);
        }
        consumer.shutdown();
    }
}
 
Example 21
Source Project: rocketmq   Source File: PullMessageProcessorTest.java    License: Apache License 2.0 5 votes vote down vote up
static ConsumerData createConsumerData(String group, String topic) {
    ConsumerData consumerData = new ConsumerData();
    consumerData.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    consumerData.setConsumeType(ConsumeType.CONSUME_PASSIVELY);
    consumerData.setGroupName(group);
    consumerData.setMessageModel(MessageModel.CLUSTERING);
    Set<SubscriptionData> subscriptionDataSet = new HashSet<>();
    SubscriptionData subscriptionData = new SubscriptionData();
    subscriptionData.setTopic(topic);
    subscriptionData.setSubString("*");
    subscriptionData.setSubVersion(100L);
    subscriptionDataSet.add(subscriptionData);
    consumerData.setSubscriptionDataSet(subscriptionDataSet);
    return consumerData;
}
 
Example 22
Source Project: DDMQ   Source File: ConsumeMessageOrderlyService.java    License: Apache License 2.0 5 votes vote down vote up
public void start() {
    if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {
        scheduledFuture = this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                ConsumeMessageOrderlyService.this.lockMQPeriodically();
            }
        }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
    }
}
 
Example 23
Source Project: DDMQ   Source File: ConsumeMessageOrderlyService.java    License: Apache License 2.0 5 votes vote down vote up
public void start() {
    if (MessageModel.CLUSTERING.equals(ConsumeMessageOrderlyService.this.defaultMQPushConsumerImpl.messageModel())) {
        scheduledFuture = this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                ConsumeMessageOrderlyService.this.lockMQPeriodically();
            }
        }, 1000 * 1, ProcessQueue.REBALANCE_LOCK_INTERVAL, TimeUnit.MILLISECONDS);
    }
}
 
Example 24
Source Project: DDMQ   Source File: ConsumeMessageOrderlyService.java    License: Apache License 2.0 5 votes vote down vote up
public void shutdown() {
    this.stopped = true;
    if (scheduledFuture != null) {
        scheduledFuture.cancel(false);
    }
    if (this.defaultMQPushConsumer.getSharedThreadPoolExecutor() == null) {
        this.consumeExecutor.shutdown();
    }
    if (this.defaultMQPushConsumer.getSharedScheduledExecutorService() == null) {
        this.scheduledExecutorService.shutdown();
    }
    if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        this.unlockAllMQ();
    }
}
 
Example 25
Source Project: J2Cache   Source File: RocketMQClusterPolicy.java    License: Apache License 2.0 5 votes vote down vote up
public RocketMQClusterPolicy(Properties props) {
    this.hosts = props.getProperty("hosts");
    String groupName = props.getProperty("name", "j2cache");
    this.topic = props.getProperty("topic", "j2cache");

    this.producer = new DefaultMQProducer(groupName);
    this.producer.setNamesrvAddr(this.hosts);

    this.consumer = new DefaultMQPushConsumer(groupName);
    this.consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_FIRST_OFFSET);
    this.consumer.setNamesrvAddr(this.hosts);
    this.consumer.setMessageModel(MessageModel.BROADCASTING);
}
 
Example 26
Source Project: DDMQ   Source File: ConsumerGroupInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ConsumerGroupInfo(String groupName, ConsumeType consumeType, MessageModel messageModel,
    ConsumeFromWhere consumeFromWhere) {
    this.groupName = groupName;
    this.consumeType = consumeType;
    this.messageModel = messageModel;
    this.consumeFromWhere = consumeFromWhere;
}
 
Example 27
Source Project: DDMQ   Source File: ConsumerGroupInfo.java    License: Apache License 2.0 5 votes vote down vote up
public boolean updateChannel(final ClientChannelInfo infoNew, ConsumeType consumeType,
    MessageModel messageModel, ConsumeFromWhere consumeFromWhere) {
    boolean updated = false;
    this.consumeType = consumeType;
    this.messageModel = messageModel;
    this.consumeFromWhere = consumeFromWhere;

    ClientChannelInfo infoOld = this.channelInfoTable.get(infoNew.getChannel());
    if (null == infoOld) {
        ClientChannelInfo prev = this.channelInfoTable.put(infoNew.getChannel(), infoNew);
        if (null == prev) {
            log.info("new consumer connected, group: {} {} {} channel: {}", this.groupName, consumeType,
                messageModel, infoNew.toString());
            updated = true;
        }

        infoOld = infoNew;
    } else {
        if (!infoOld.getClientId().equals(infoNew.getClientId())) {
            log.error("[BUG] consumer channel exist in broker, but clientId not equal. GROUP: {} OLD: {} NEW: {} ",
                this.groupName,
                infoOld.toString(),
                infoNew.toString());
            this.channelInfoTable.put(infoNew.getChannel(), infoNew);
        }
    }

    this.lastUpdateTimestamp = System.currentTimeMillis();
    infoOld.setLastUpdateTimestamp(this.lastUpdateTimestamp);

    return updated;
}
 
Example 28
Source Project: DDMQ   Source File: MQHelper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reset consumer topic offset according to time
 *
 * @param messageModel which model
 * @param instanceName which instance
 * @param consumerGroup consumer group
 * @param topic topic
 * @param timestamp time
 */
public static void resetOffsetByTimestamp(
    final MessageModel messageModel,
    final String instanceName,
    final String consumerGroup,
    final String topic,
    final long timestamp) throws Exception {
    final Logger log = ClientLogger.getLog();

    DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup);
    consumer.setInstanceName(instanceName);
    consumer.setMessageModel(messageModel);
    consumer.start();

    Set<MessageQueue> mqs = null;
    try {
        mqs = consumer.fetchSubscribeMessageQueues(topic);
        if (mqs != null && !mqs.isEmpty()) {
            TreeSet<MessageQueue> mqsNew = new TreeSet<MessageQueue>(mqs);
            for (MessageQueue mq : mqsNew) {
                long offset = consumer.searchOffset(mq, timestamp);
                if (offset >= 0) {
                    consumer.updateConsumeOffset(mq, offset);
                    log.info("resetOffsetByTimestamp updateConsumeOffset success, {} {} {}",
                        consumerGroup, offset, mq);
                }
            }
        }
    } catch (Exception e) {
        log.warn("resetOffsetByTimestamp Exception", e);
        throw e;
    } finally {
        if (mqs != null) {
            consumer.getDefaultMQPullConsumerImpl().getOffsetStore().persistAll(mqs);
        }
        consumer.shutdown();
    }
}
 
Example 29
Source Project: rocketmq   Source File: RebalancePushImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 移除不需要的队列相关的信息
 * 1. 持久化消费进度,并移除之
 * 2. 顺序消费&集群模式,解锁对该队列的锁定
 *
 * @param mq 消息队列
 * @param pq 消息处理队列
 * @return 是否移除成功
 */
@Override
public boolean removeUnnecessaryMessageQueue(MessageQueue mq, ProcessQueue pq) {
    // 同步队列的消费进度,并移除之。
    this.defaultMQPushConsumerImpl.getOffsetStore().persist(mq);
    this.defaultMQPushConsumerImpl.getOffsetStore().removeOffset(mq);
    // 集群模式下,顺序消费移除时,解锁对队列的锁定
    if (this.defaultMQPushConsumerImpl.isConsumeOrderly() && MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        try {
            // 有序消费模式下,ConsumeMessageOrderlyService在执行 MessageListenerOrderly#consumeMessage() 时会锁定ProcessQueue,
            // 执行此方法完后才会释放ProcessQueue的Lock,见ConsumeMessageOrderlyService#464 行
            // 锁定不成功说明在长达1S的时间内一直都在执行 MessageListenerOrderly#consumeMessage 方法
            // 也就是说 consumeMessage() 方法执行时间超过1S,每条消息的消费速度以及方法参数List<MessageExt>的大小可能存在问题
            // 此时只能等待相应MessageQueue在Broker的锁定时间过期(1m)后再被其他消费者锁定
            if (pq.getLockConsume().tryLock(1000, TimeUnit.MILLISECONDS)) {
                try {
                    return this.unlockDelay(mq, pq);
                } finally {
                    pq.getLockConsume().unlock();
                }
            } else {
                log.warn("[WRONG]mq is consuming, so can not unlock it, {}. maybe hanged for a while, {}", mq, pq.getTryUnlockTimes());
                pq.incTryUnlockTimes();
            }
        } catch (Exception e) {
            log.error("removeUnnecessaryMessageQueue Exception", e);
        }

        return false;
    }
    return true;
}
 
Example 30
Source Project: DDMQ   Source File: ConsumeMessageOrderlyService.java    License: Apache License 2.0 5 votes vote down vote up
public void shutdown() {
    this.stopped = true;
    if (scheduledFuture != null) {
        scheduledFuture.cancel(false);
    }
    if (this.defaultMQPushConsumer.getSharedThreadPoolExecutor() == null) {
        this.consumeExecutor.shutdown();
    }
    if (this.defaultMQPushConsumer.getSharedScheduledExecutorService() == null) {
        this.scheduledExecutorService.shutdown();
    }
    if (MessageModel.CLUSTERING.equals(this.defaultMQPushConsumerImpl.messageModel())) {
        this.unlockAllMQ();
    }
}