/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.rocketmq.broker.latency; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.rocketmq.broker.BrokerController; import org.apache.rocketmq.common.ThreadFactoryImpl; import org.apache.rocketmq.common.constant.LoggerName; import org.apache.rocketmq.logging.InternalLogger; import org.apache.rocketmq.logging.InternalLoggerFactory; import org.apache.rocketmq.remoting.netty.RequestTask; import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; /** * BrokerFastFailure will cover {@link BrokerController#sendThreadPoolQueue} and * {@link BrokerController#pullThreadPoolQueue} * BrokerFastFail线程池。这个主要是Broker进行各项流控的一个定时JOB * @author ; */ public class BrokerFastFailure { private static final InternalLogger log = InternalLoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); /** * 周期性调度的线程池 */ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl( "BrokerFastFailureScheduledThread")); private final BrokerController brokerController; public BrokerFastFailure(final BrokerController brokerController) { this.brokerController = brokerController; } /** * 强制转换Runnable * @param runnable runnable * @return ; */ public static RequestTask castRunnable(final Runnable runnable) { try { if (runnable instanceof FutureTaskExt) { FutureTaskExt object = (FutureTaskExt) runnable; return (RequestTask) object.getRunnable(); } } catch (Throwable e) { log.error(String.format("castRunnable exception, %s", runnable.getClass().getName()), e); } return null; } /** * 10秒钟执行一次 */ public void start() { this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { @Override public void run() { if (brokerController.getBrokerConfig().isBrokerFastFailureEnable()) { cleanExpiredRequest(); } } }, 1000, 10, TimeUnit.MILLISECONDS); } /** * 10秒钟clean超时的请求 */ private void cleanExpiredRequest() { //如果pagecache繁忙 while (this.brokerController.getMessageStore().isOSPageCacheBusy()) { try { if (!this.brokerController.getSendThreadPoolQueue().isEmpty()) { final Runnable runnable = this.brokerController.getSendThreadPoolQueue().poll(0, TimeUnit.SECONDS); if (null == runnable) { break; } final RequestTask rt = castRunnable(runnable); //返回客户端系统繁忙,启动流控 rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[PCBUSY_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", System.currentTimeMillis() - rt.getCreateTimestamp(), this.brokerController.getSendThreadPoolQueue().size())); } else { break; } } catch (Throwable ignored) { } } cleanExpiredRequestInQueue(this.brokerController.getSendThreadPoolQueue(), this.brokerController.getBrokerConfig().getWaitTimeMillsInSendQueue()); cleanExpiredRequestInQueue(this.brokerController.getPullThreadPoolQueue(), this.brokerController.getBrokerConfig().getWaitTimeMillsInPullQueue()); cleanExpiredRequestInQueue(this.brokerController.getHeartbeatThreadPoolQueue(), this.brokerController.getBrokerConfig().getWaitTimeMillsInHeartbeatQueue()); cleanExpiredRequestInQueue(this.brokerController.getEndTransactionThreadPoolQueue(), this .brokerController.getBrokerConfig().getWaitTimeMillsInTransactionQueue()); } /** * 在各个队列里清除超时的请求,并返回给客户端系统繁忙 * @param blockingQueue 队列 * @param maxWaitTimeMillsInQueue 超时 */ void cleanExpiredRequestInQueue(final BlockingQueue<Runnable> blockingQueue, final long maxWaitTimeMillsInQueue) { while (true) { try { if (!blockingQueue.isEmpty()) { final Runnable runnable = blockingQueue.peek(); if (null == runnable) { break; } final RequestTask rt = castRunnable(runnable); if (rt == null || rt.isStopRun()) { break; } final long behind = System.currentTimeMillis() - rt.getCreateTimestamp(); if (behind >= maxWaitTimeMillsInQueue) { if (blockingQueue.remove(runnable)) { rt.setStopRun(true); rt.returnResponse(RemotingSysResponseCode.SYSTEM_BUSY, String.format("[TIMEOUT_CLEAN_QUEUE]broker busy, start flow control for a while, period in queue: %sms, size of queue: %d", behind, blockingQueue.size())); } } else { break; } } else { break; } } catch (Throwable ignored) { } } } /** * 停止线程池 */ public void shutdown() { this.scheduledExecutorService.shutdown(); } }