package com.jd.binlog.work;

import com.jd.binlog.alarm.AlarmUtils;
import com.jd.binlog.alarm.utils.JDMailParas;
import com.jd.binlog.alarm.utils.JDPhoneParas;
import com.jd.binlog.conn.MySQLConnector;
import com.jd.binlog.conn.MySQLExecutor;
import com.jd.binlog.convert.Converter;
import com.jd.binlog.convert.TableRowsParser;
import com.jd.binlog.dbsync.*;
import com.jd.binlog.dbsync.event.*;
import com.jd.binlog.dump.BinlogDump;
import com.jd.binlog.dump.DumpType;
import com.jd.binlog.exception.BinlogException;
import com.jd.binlog.exception.ErrorCode;
import com.jd.binlog.filter.aviater.AviaterRegexFilter;
import com.jd.binlog.inter.alarm.IAlarm;
import com.jd.binlog.inter.msg.IConvert;
import com.jd.binlog.inter.msg.IRepartition;
import com.jd.binlog.inter.produce.IProducer;
import com.jd.binlog.inter.rule.IRule;
import com.jd.binlog.inter.work.IBinlogHandler;
import com.jd.binlog.inter.work.IBinlogWorker;
import com.jd.binlog.inter.zk.ILeaderSelector;
import com.jd.binlog.meta.Meta;
import com.jd.binlog.meta.MetaInfo;
import com.jd.binlog.mysql.EOFPacket;
import com.jd.binlog.mysql.ErrorPacket;
import com.jd.binlog.net.BufferPool;
import com.jd.binlog.parser.SimpleDdlParser;
import com.jd.binlog.parser.TableMeta;
import com.jd.binlog.parser.TableMetaCache;
import com.jd.binlog.performance.PerformanceUtils;
import com.jd.binlog.producer.Producer;
import com.jd.binlog.protocol.WaveEntry;
import com.jd.binlog.rule.MQRule;
import com.jd.binlog.util.CharUtils;
import com.jd.binlog.util.LogUtils;
import com.jd.binlog.util.TimeUtils;
import org.apache.commons.lang.StringUtils;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import java.util.*;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.ReentrantLock;

import static com.jd.binlog.inter.msg.IConvert.BEGIN;
import static com.jd.binlog.inter.msg.IConvert.COMMIT;
import static com.jd.binlog.util.GTIDUtils.compare;

/**
 * Created on 18-5-12
 *
 * @author pengan
 */
public class BinlogWorker extends Thread implements IBinlogWorker {
    // 所有的发送规则是 乱序
    private boolean isMess = false;

    // read buffer offset
    private volatile int offset = 0;

    // last read buffer that not end with 1 package
    private ByteBuffer lastBuffer;

    // binlog file only exist in rotate event so have
    private volatile String binlogFile = "";

    // is terminal gtid
    private volatile boolean isTerminalGtid;

    // is closed atomic boolean flag to mark the MySQL connection dump is closed
    private final AtomicBoolean isClosed = new AtomicBoolean(false);

    // 操作 log position 队列锁
    private ReentrantLock lock = new ReentrantLock();

    // 由于有限流 所以队列长度不会太大 最大长度 processors + bufferSize
    private ConcurrentLinkedQueue<LogPosition> logPositions = new ConcurrentLinkedQueue<LogPosition>();

    // gtids
    private Map<String, GTID> gtids = new HashMap<String, GTID>();

    // 过滤 pt工具表生成器
    private AviaterRegexFilter ptOnlineTableFilter = new AviaterRegexFilter("^_.*_old$,^_.*_new$");

    // dump byte buffer pool 传入参数
    private BufferPool bufferPool;

    // MySQL meta info 传入参数
    private MetaInfo metaInfo;

    // dump MySQL 实例 host:port
    private String host;

    // transaction id
    private long trxID = 0;

    // monitor map
    private Map<String, Object> monitor;

    // MySQL dump connection
    private MySQLConnector dump;

    // Binlog decoder
    private LogDecoder decoder; // init

    // Binlog Context
    private LogContext context; // init

    // leader selector 循环引用 无法造成fgc 传入参数
    private ILeaderSelector leader;

    // TableMeta cache contains table meta information {field type and field value}
    // 原生 binlog当中的表名 以及库名 不要更新大小写
    private TableMetaCache tableMetaCache;

    // 初始化的log position
    private LogPosition originPos;

    // rule list
    private List<IRule> rs;

    // rule number
    private int ruleNum;

    // partition function
    private IRepartition part;

    // binlog handler for filter data and send message
    private IBinlogHandler handler;

    // throttler queue for binlog dump 阻流器
    private LinkedBlockingQueue<Object> throttler = new LinkedBlockingQueue<Object>();

    /**
     * 启动binlog worker
     *
     * @param partition
     * @param throttleSize 流控 buffer 大小
     * @param bufferPool
     * @param metaInfo
     * @param leader
     * @throws Exception
     */
    public BinlogWorker(int partition,
                        int throttleSize,
                        BufferPool bufferPool,
                        MetaInfo metaInfo,
                        ILeaderSelector leader,
                        ArrayList<LinkedBlockingQueue<IRule.RetValue>> queues) throws Exception {
        super(metaInfo.getHost());

        // 每个事件可能都需要使用 所以初始化1次
        this.leader = leader;
        this.metaInfo = metaInfo;
        this.bufferPool = bufferPool;
        this.host = metaInfo.getHost() + ":" + metaInfo.getPort();

        // init part
        this.part = initPartition(partition, queues);

        this.rs = new ArrayList<IRule>();
        for (Meta.Rule rule : metaInfo.getDbInfo().getRule()) {
            switch (rule.getStorage()) {
                case KV_STORAGE:
                    break;
                case MQ_STORAGE:
                    isMess &= initMQRule(rule);
                    break;
            }
        }
        // 常用 不再重复计算
        this.ruleNum = rs.size();

        if (this.ruleNum == 0) {
            // 如果规则不存在 则直接放弃dump 数据抛出异常 进行重试
            throw new BinlogException(ErrorCode.ERR_RULE_EMPTY, new Exception("过滤规则为空"));
        }

        fillThrottler(throttleSize);

        // MySQL connector 管理端已经验证 正常来说应该能够支持
        this.dump = new MySQLConnector(metaInfo.getDbInfo());

        // log position
        this.logPositions = new ConcurrentLinkedQueue<>();

        // table meta cache using guava cache
        this.tableMetaCache = new TableMetaCache(new MySQLConnector(metaInfo.getDbInfo()));

        // 分配内存
        this.lastBuffer = bufferPool.allocate();

        this.handler = new BinlogHandler(rs, host, isMess, this);

        // init monitor for capacity 4 size
        this.monitor = new HashMap<>(4);

        // remember the current timestamp
        monitor.put("start", TimeUtils.time());
    }

    /**
     * 填充 流控器
     *
     * @param throttleSize
     */
    private void fillThrottler(int throttleSize) {
        for (int i = 0; i < throttleSize; i++) {
            throttler.offer(object);
        }
    }


    /**
     * 初始化 mq 规则
     *
     * @param rule
     * @return isMess 是否是乱序 很有可能出现异常  所以提前
     * @throws Exception
     */
    private boolean initMQRule(Meta.Rule rule) throws Exception {
        Meta.MQRule mqRule = Meta.MQRule.unmarshalJson(rule.getAny());

        // producer implement
        IProducer producer = Producer.initProducer(mqRule.getProducerClass(), mqRule.getPara(), mqRule.getTopic());

        // convert implement
        IConvert conv = Converter.initConverter(rule.getConvertClass());

        MQRule mr = new MQRule(mqRule, conv, producer, part, this);
        rs.add(mr);

        return mqRule.getOrder() == Meta.OrderType.NO_ORDER;
    }

    /**
     * 依据partition 初始化 queue
     *
     * @param partition
     * @param queues
     * @return IRepartition
     */
    private IRepartition initPartition(int partition,
                                       final List<LinkedBlockingQueue<IRule.RetValue>> queues) {
        LogUtils.info.info("queue number is " + partition);

        return retValue -> {
            // never will the retValue is null
            queues.get(retValue.partition % partition).offer(retValue);
        };
    }

    @Override
    public void run() {
        try {
            // prepare
            prepare();

            // read from socket channel
            read();
        } catch (InterruptedException e) {
            // InterruptedException that means all have closed because of the interrupt
            close(); // just close
        } catch (BinlogException e) {
            handleException(e);
        } catch (Throwable e) {
            handleException(new BinlogException(ErrorCode.ERR_UNKNOWN, e));
        }
    }

    @Override
    public boolean close() {
        if (!isClosed.compareAndSet(false, true)) {
            LogUtils.debug.warn("already closed");
            return false;
        }
        LogUtils.info.info("close handler");

        long connectionId = dump.getConnectionId();
        LogUtils.info.info(host + " close MySQL connection " + connectionId);

        try {
            dump.disconnect();
            dump = null;
        } catch (IOException e) {
            LogUtils.error.error("MySQL " + host + " connect error ", e);
        }

        // using the already connected Connection
        try {
            if (connectionId != 0) {
                tableMetaCache.kill(connectionId);
            }
        } catch (Throwable exp) {
            LogUtils.error.error("close " + host + " connection error ", exp);
        }

        LogUtils.info.info("close table meta cache");
        try {
            tableMetaCache.close();
            tableMetaCache = null;
        } catch (Throwable exp) {
            LogUtils.error.error("close " + host + " table meta cache error ", exp);
        }

        LogUtils.info.info("close handler thread");
        // close handler
        if (handler != null) {
            handler.clear();
        }
        handler = null;

        // 清空流控器
        throttler.clear();
        throttler = null;

        // 置空 去除悬挂引用 防止gc 失败问题 需要重新获取数据
        logPositions.clear();
        logPositions = null;

        gtids.clear();
        gtids = null;

        ptOnlineTableFilter = null;

        bufferPool.recycle(lastBuffer);
        lastBuffer = null;
        bufferPool = null;

        decoder = null;
        context.clearAllTables();
        context = null;

        originPos.clear();
        originPos = null;

        for (IRule rule : rs) {
            // 关闭 规则
            rule.close();
        }
        rs.clear();
        rs = null;

        part = null;

        monitor.clear();
        monitor = null;

        try {
            /**
             * 当前线程 不是发起close线程
             */
            this.interrupt();
        } catch (Throwable exp) {
            LogUtils.error.error("binlog worker interrupt", exp);
        }
        return true;
    }

    @Override
    public boolean isClosed() {
        return isClosed.get();
    }

    /**
     * 打印binlog 位置日志
     *
     * @param logPositions
     */
    private void debugLogPosition(ConcurrentLinkedQueue<LogPosition> logPositions) {
        if (LogUtils.debug.isDebugEnabled()) {
            Iterator<LogPosition> liter = logPositions.iterator();
            boolean isHead = true;
            int count = 0;
            while (liter.hasNext()) {
                LogPosition lp = liter.next();
                if (isHead) {
                    LogUtils.debug.debug(host + " truncLogPosQueue logPositions head is " + lp);
                    isHead = false;
                }
                count++;
            }
            LogUtils.debug.debug(host + " truncLogPosQueue logPositions queue size " + count);

            BlockingQueue<Object> queue = this.throttler;
            if (queue != null) {
                LogUtils.debug.debug(host + " throttler queue size " + queue.size());
            }
        }
    }

    /**
     * 合并commit 位置
     * ----------------------|    |--------------------|    |--------------------|
     * node1.isCommit = true | -> | node2.isCommit=true| -> |node3.isCommit=false| ...
     * ----------------------|    |--------------------|    |--------------------|
     * <p>
     * then the result remove 1node leave 2node... behind
     */
    private void truncLogPosQueue(ConcurrentLinkedQueue<LogPosition> logPositions) {
        if (logPositions.isEmpty()) {
            LogUtils.warn.warn("no binlog position object in queue");
            return;
        }

        // 根据concurrent list 实现方式 一次size 相当于 直接遍历一遍 链表
        LogPosition curr = null;
        LogPosition pre = null;

        LinkedList<LogPosition> rms = new LinkedList<LogPosition>();
        Iterator<LogPosition> iterator = logPositions.iterator();

        while (iterator.hasNext()) {
            if (pre == null) {
                pre = iterator.next();
                continue;
            }
            curr = iterator.next();

            if (pre.isCommit() && curr.isCommit()) {
                rms.add(pre);
            }
            pre = curr;
        }

        removeQueueWithLock(logPositions, rms);

        // 轻易不要开work日志
        debugLogPosition(logPositions);
    }

    /**
     * 如果未获取到锁 可能有冲突 立马返回 等待下次执行
     *
     * @param logPositions binlog 位置队列 {线程安全}
     * @param rms          需要被删除的 对象
     */
    private void removeQueueWithLock(ConcurrentLinkedQueue<LogPosition> logPositions,
                                     List<LogPosition> rms) {
        if (lock.tryLock()) {
            try {
                // 删除队列 需要有锁防止写冲突 注意这里的log pos 与 work.removeLogPosition() 为不同属性
                for (LogPosition lp : rms) {
                    logPositions.remove(lp);
                    keepDump();
                }
            } finally {
                lock.unlock();
            }
        }
        rms.clear();
    }

    @Override
    public LogPosition getLatestLogPosWithRm() {
        ConcurrentLinkedQueue<LogPosition> logPositions = this.logPositions;
        if (logPositions == null) {
            return null;
        }

        // 轻易不要开work日志
        debugLogPosition(logPositions);

        LogPosition curr;
        LogPosition pre = null;
        int len = 0;

        List<LogPosition> rms = new LinkedList<>();
        // 避免进入无线循环当中 所以控制次数  不需要担心队列过长 因为有truncate Log position 保证
        Iterator<LogPosition> iter = logPositions.iterator();
        while (iter.hasNext() && (curr = iter.next()) != null
                && len++ < THROTTLE_QUEUE_SIZE) {
            if (curr.isCommit()) {
                rms.add(curr); // 添加到 删除队列
                pre = curr;
                continue;
            }
            break; // 如果不是commit 直接退出
        }

        removeQueueWithLock(logPositions, rms);

        if (pre != null) {
            pre.mergeOriginLogPos(originPos);
            pre.refresh();
            return pre;
        }
        return null;
    }

    @Override
    public LogPosition getLatestLogPos() {
        ConcurrentLinkedQueue<LogPosition> logPositions = this.logPositions;
        if (logPositions == null) {
            return null;
        }

        debugLogPosition(logPositions);

        LogPosition curr;
        LogPosition pre = null;
        int len = 0;

        // 避免进入无线循环当中 所以控制次数  不需要担心队列过长 因为有truncate Log position 保证
        Iterator<LogPosition> iter = logPositions.iterator();
        while (iter.hasNext() && (curr = iter.next()) != null
                && len++ < THROTTLE_QUEUE_SIZE) {
            if (curr.isCommit()) {
                pre = curr;
                continue;
            }
            break; // 如果不是commit 直接退出
        }

        if (pre != null) {
            LogPosition np = pre.clone();
            np.mergeOriginLogPos(originPos);
            np.refresh();
            return np;
        }
        return null;
    }

    @Override
    public void handleException(BinlogException exp) {
        LogUtils.debug.debug("handleException stop dump service");

        LogUtils.error.error("host: " + host + " io exception ", exp);

        boolean success = false;

        // 获取最新的binlog位置 在io异常情况下 将这个位置同步到zookeeper
        try {
            leader.refreshLogPos();
        } catch (Exception e) {
            LogUtils.error.error("refresh log position error ", e);
        }

        try {
            success = close();
        } catch (Throwable e) {
            LogUtils.error.error("close session error", e);
        }

        // increase retryTimes meta and push to zookeeper
        ILeaderSelector leader = this.leader;
        MetaInfo metaInfo = this.metaInfo;
        if (success && leader != null && metaInfo != null) {
            // get meta info
            byte[] errMsg = exp.message(metaInfo.getBinlogInfo().getLeader(), host, metaInfo.getHost());
            Meta.Error err = new Meta.Error().
                    setCode(exp.getErrorCode().errorCode).
                    setMsg(errMsg);

            LogUtils.warn.warn("add session retry times");
            switch (exp.getErrorCode().according()) {
                case Retry:
                    metaInfo.addSessionRetryTimes();
                    metaInfo.setError(err);
                    // 重试报警
                    AlarmUtils.mail(metaInfo.getRetryTimes(), metaInfo.getAlarm().getRetry(),
                            JDMailParas.mailParas(
                                    MetaInfo.mailTo(metaInfo.getAlarm().getUsers()),
                                    String.format(IAlarm.MailSubTemplate, host, metaInfo.getHost()),
                                    errMsg));
                    break;
                case Stop:
                    metaInfo.fillRetryTimes();
                    metaInfo.setError(err);
                    // 系统报警
                    AlarmUtils.phone(JDPhoneParas.phoneParas(errMsg));
                    break;
            }


            try {
                leader.updateCounter(metaInfo);
            } catch (Exception e) {
                LogUtils.error.error("update counter error", e);
            }

            LogUtils.warn.warn("abandon leader ship");
            leader.abandonLeaderShip();

            // only close success then have the lock to reset null
            this.leader = null;
            this.metaInfo = null;
        }
    }

    @Override
    public void handleError(Throwable exp) {
        LogUtils.error.error("host: " + host + " error ", exp);
        close();
    }

    @Override
    public void removeLogPosition(LogPosition target) {
        ConcurrentLinkedQueue<LogPosition> logPositions = this.logPositions;
        if (logPositions == null) {
            return;
        }

        // 判断引用基数是否为 0
        int counter = target.decrementAndGet();

        if (LogUtils.debug.isDebugEnabled()) {
            LogUtils.debug.debug("quote counter is " + counter + ", log position " + target);
        }

        if (counter == 0) {
            logPositions.remove(target);
            // 如果计数器 != 0 则不能往 阻流器 当中填充object 因为有可能会导致队列无限大 1 -> 16 
            keepDump();
        }
    }

    public void read() throws IOException, InterruptedException {
        ByteBuffer buffer;

        SocketChannel channel = dump.getChannel();

        // read bytes from channel into buffer
        while (!isClosed() && !isInterrupted() &&
                (buffer = this.lastBuffer) != null) { // continue when queue have at least 1 object or else will block

            int got = channel.read(buffer);
            if (got < 0) {
                LogUtils.error.error(host + " " + " get < 0 !!!");
                throw new IOException("got < 0");
            }

            TableMetaCache meta = this.tableMetaCache;
            ILeaderSelector leader = this.leader;
            Map<String, GTID> gtids = this.gtids;
            ConcurrentLinkedQueue<LogPosition> logPositions = this.logPositions;
            LogDecoder decoder = this.decoder;
            LinkedBlockingQueue<Object> throttler = this.throttler;
            if (meta == null
                    || leader == null
                    || gtids == null
                    || logPositions == null
                    || decoder == null
                    || throttler == null) {
                return;
            }
            // prevent NullPointerException
            handlePackage(buffer, meta, leader, gtids, logPositions, decoder, throttler);
        }
    }

    @Override
    public void startWorking() {
        this.start(); // thread start
        this.handler.startHandler(); // startWorking handler thread
    }

    /**
     * open the valve to keep on dump data
     */
    public void keepDump() {
        LinkedBlockingQueue<Object> thq = throttler;
        if (thq != null) {
            thq.offer(object);
        }
    }

    @Override
    public Map<String, Object> monitor() {
        return this.monitor;
    }

    /**
     * handle byte buffer packet
     *
     * @param buffer
     * @param meta
     * @param leader
     * @param gtids
     * @param logPositions
     * @param decoder
     */
    private final void handlePackage(ByteBuffer buffer,
                                     TableMetaCache meta,
                                     ILeaderSelector leader,
                                     Map<String, GTID> gtids,
                                     ConcurrentLinkedQueue<LogPosition> logPositions,
                                     LogDecoder decoder,
                                     LinkedBlockingQueue<Object> throttler) throws InterruptedException {
        int offset = this.offset, position = buffer.position(), length;

        while (!isClosed()) {
            length = getPacketLength(buffer, offset);

            if (length == -1) {// 未达到可计算数据包长度的数据
                if (!buffer.hasRemaining()) {
                    checkReadBuffer(buffer, offset, position);
                }
                break;
            }

            if (position >= offset + length) {
                // 提取一个数据包的数据进行处理
                buffer.position(offset);
                byte[] data = new byte[length];

                buffer.get(data, 0, length);
                decodeBytes(data, meta, leader, gtids, logPositions, decoder, throttler);

                // 设置偏移量
                offset += length;

                if (position == offset) {// 数据正好全部处理完毕
                    if (this.offset != 0) {
                        this.offset = 0;
                    }
                    buffer.clear();
                    break;
                } else {// 还有剩余数据未处理
                    this.offset = offset;
                    buffer.position(position);
                    continue;
                }
            } else {// 未到达一个数据包的数据
                if (!buffer.hasRemaining()) {
                    checkReadBuffer(buffer, offset, position);
                }
                break;
            }
        }
    }

    /**
     * get packet length using byte buffer
     *
     * @param buffer
     * @param offset
     * @return
     */
    private final int getPacketLength(ByteBuffer buffer, int offset) {
        if (buffer.position() < offset + PACKET_HEAD_SIZE) {
            return -1;
        } else {
            int length = buffer.get(offset) & 0xff;
            length |= (buffer.get(++offset) & 0xff) << 8;
            length |= (buffer.get(++offset) & 0xff) << 16;
            return length + PACKET_HEAD_SIZE;
        }
    }

    /**
     * 检查ReadBuffer容量,不够则扩展当前缓存,直到最大值。
     */
    private final ByteBuffer checkReadBuffer(ByteBuffer buffer, int offset, int position) {
        // 当偏移量为0时需要扩容,否则移动数据至偏移量为0的位置。
        if (offset == 0) {
            if (buffer.capacity() >= MAX_PACKET_SIZE) {
                throw new IllegalArgumentException("Packet size over the limit.");
            }

            LogUtils.warn.warn("checkReadBuffer==2, offset: " + offset + ", position: " + position + ", remaining: " + buffer.remaining());

            int size = buffer.capacity() << 1;
            size = (size > MAX_PACKET_SIZE) ? MAX_PACKET_SIZE : size;
            ByteBuffer newBuffer = ByteBuffer.allocate(size);
            buffer.position(offset);
            newBuffer.put(buffer);
            this.lastBuffer = newBuffer;

            // 回收扩容前的缓存块
            bufferPool.recycle(buffer);

            return newBuffer;
        } else {

            buffer.position(offset);
            buffer.compact();
            this.offset = 0;
            return buffer;
        }
    }

    /**
     * here packet is get success
     *
     * @param data
     * @param meta
     * @param leader
     * @param gtids
     * @param logPositions
     * @param decoder
     * @param throttler
     * @throws InterruptedException
     */
    private final void decodeBytes(byte[] data,
                                   TableMetaCache meta, ILeaderSelector leader,
                                   Map<String, GTID> gtids,
                                   ConcurrentLinkedQueue<LogPosition> logPositions,
                                   LogDecoder decoder,
                                   LinkedBlockingQueue<Object> throttler) throws BinlogException, InterruptedException {
        throttler.take(); // 从阻流器当中取出一个

        long dumpTime = TimeUtils.time();
        LogEvent event;
        switch (data[4]) {
            case EOFPacket.FIELD_COUNT:
                // just an eof no need to worry
                LogUtils.debug.debug("eof read from master");

                keepDump();
                // offer object to throttler
                return;
            case ErrorPacket.FIELD_COUNT:
                ErrorPacket err = new ErrorPacket();
                err.read(data, 4);
                throw new BinlogException(ErrorCode.valueOfMySQLErrno(err.errno), new Exception(new String(err.message)), binlogFile + ":4");
            default:
                LogUtils.debug.debug("continue");
        }

        int origin = PACKET_HEAD_SIZE + 1;
        int limit = data.length - origin;
        LogBuffer buffer = new LogBuffer(data, origin, limit);
        event = decoder.decode(buffer, context);

        if (event == null) {
            LogUtils.warn.warn("event is null");

            keepDump();
            return;
        }

        // calculate dump delay
        this.monitor.put(PerformanceUtils.DUMP_DELAY_KEY, event.getWhen() - dumpTime);

        // 日志记录当前延迟时间
        PerformanceUtils.perform(PerformanceUtils.DUMP_DELAY_KEY, event.getWhen(), dumpTime);
        PerformanceUtils.perform(PerformanceUtils.DECODE_DELAY_KEY, event.getWhen());

        Carrier carrier = new Carrier(binlogFile, host, event);

        truncLogPosQueue(logPositions);
        try {
            // 阻塞以免避免任务队列过长
            boolean callKeepDump = handle(event, carrier, meta, leader, gtids, logPositions);
            if (callKeepDump) {
                keepDump();
            }
        } catch (BinlogException e) {
            handleException(e);
        } catch (Throwable e) {
            handleError(e);
        }
    }


    /**
     * handle event which data is stored into carrier 根据某些字段 并行
     *
     * @param event
     * @param carrier
     * @param meta
     * @param leader
     * @param gtids
     * @param logPositions @throws Exception
     * @return boolean whether to call keepDump
     */
    private boolean handle(LogEvent event, Carrier carrier, TableMetaCache meta,
                           ILeaderSelector leader, Map<String, GTID> gtids,
                           ConcurrentLinkedQueue<LogPosition> logPositions) throws BinlogException {
        int eventType = event.getHeader().getType();
        boolean isCommit = false;

        switch (eventType) {
            case LogEvent.ROTATE_EVENT:
                // 更新当前event的binlog file
                binlogFile = ((RotateLogEvent) event).getFilename(); // 只需要更新当前binlog file name
                return true;
            case LogEvent.QUERY_EVENT:
                // ddl语句 可能需要更新元数据信息 刷新指定 tableMeta
                isCommit = updateTableMetaByDDL((QueryLogEvent) event, meta, carrier);
                break;
            case LogEvent.USER_VAR_EVENT:
            case LogEvent.INTVAR_EVENT:
            case LogEvent.RAND_EVENT:
                break;
            case LogEvent.WRITE_ROWS_EVENT_V1:
            case LogEvent.WRITE_ROWS_EVENT:
            case LogEvent.UPDATE_ROWS_EVENT_V1:
            case LogEvent.UPDATE_ROWS_EVENT:
            case LogEvent.DELETE_ROWS_EVENT_V1:
            case LogEvent.DELETE_ROWS_EVENT:
                TableMapLogEvent tme = ((RowsLogEvent) event).getTable();

                if (tme == null) {
                    throw new BinlogException(ErrorCode.ERR_DUMP_NO_TABLE, new Exception("binlog file " + binlogFile + ", binlog pos " + event.getLogPos()
                            + ", event type " + eventType + ",获取table map log event 失败 "), "");
                }

                String table = tme.getTableName();
                String db = tme.getDbName();
                // 判断是否是 是pt online 工具生成的临时表 如果是临时表所有的数据都过滤掉
                if (ptOnlineTableFilter.filter(table)) {
                    return true;
                }

                carrier.car.db = db;
                carrier.car.table = table;
                carrier.car.tableMeta = updateTableMetaByCase(tme.getColumnInfo(), db, table, meta);
                if (carrier.car.tableMeta == null) {
                    throw new BinlogException(ErrorCode.ERR_DUMP_NO_TABLE, new Exception("获取表 " + db + "." + table + "元数据信息失败"), db + "." + table);
                }

                try {
                    TableRowsParser.parse(carrier); // 到这里需要将row进行解析 到具体的数值 开始并行发送
                } catch (Throwable exp) {
                    LogUtils.error.error("表 " + db + "." + table + " 解析失败 协议不兼容/被更改", exp);
                    throw new BinlogException(ErrorCode.WARN_MySQL_ROWEVENT_PARSE, new Exception(db + "." + table + " 解析失败 协议不兼容"),
                            "file=" + binlogFile + ", pos=" + event.getLogPos()
                                    + ", type=" + eventType);
                }
                break;
            case LogEvent.GTID_LOG_EVENT:

                GtidLogEvent gtidLogEvent = (GtidLogEvent) event;
                GTID newGtid = new GTID(gtidLogEvent.getGtidGNO(), gtidLogEvent.getSid());

                if (LogUtils.debug.isDebugEnabled()) {
                    LogUtils.debug.debug("new gtid:" + newGtid.toString() + ",terminal gtid : " + metaInfo.getTerminal());
                }

                Meta.Terminal terminal = null;
                if ((terminal = metaInfo.getTerminal()) != null &&
                        compare(newGtid.toString(), terminal.getGtid()) >= 0) {
                    isTerminalGtid = true;
                }

                LogUtils.debug.debug("is terminal gtid ? " + isTerminalGtid);

                if (gtids.size() == 0) {
                    gtids.put(newGtid.sid, newGtid);
                } else {
                    addNewGtid(gtids, newGtid);
                }
                return true;
            case LogEvent.XID_EVENT:
                isCommit = true;

                // set transaction id
                trxID = ((XidLogEvent) event).getXid();
                if (LogUtils.debug.isDebugEnabled()) {
                    LogUtils.debug.debug("transaction id " + trxID);
                }
                break;
            default:
                return true;
        }

        LogPosition logPos = new LogPosition(
                binlogFile,
                event.getLogPos(),
                event.getWhen(),
                isCommit, ruleNum);
        carrier.car.logPos = logPos;
        carrier.car.trxID = trxID + 1;

        if (gtids.size() != 0) { // MySQL 5.5是没有gtid 并且高版本的MySQL也可能不开gtid
            logPos.addAllGtids(gtids);
            logPos.refresh();
        }

        // 多线程发送版本不允许强制同步binlog 位置
        logPositions.offer(logPos);

        // 日志记录解析时间
        PerformanceUtils.perform(PerformanceUtils.PARSE_DELAY_KEY, event.getWhen());

        // 放入队列
        IBinlogHandler handler = this.handler;
        if (handler != null) {
            handler.offer(carrier);
        }

        return false;
    }

    /**
     * 根据ddl变更表结构信息
     *
     * @return isCommit
     */
    private final boolean updateTableMetaByDDL(QueryLogEvent qe, TableMetaCache tableMetaCache, Carrier carrier) {
        String query = qe.getQuery();
        String upper = CharUtils.toUpperCase(query);

        // 非commit or begin 前提下
        if (StringUtils.equals(upper, COMMIT)) {
            trxID += 1;
            if (LogUtils.debug.isDebugEnabled()) {
                LogUtils.debug.debug("commit sql transaction id " + trxID);
            }
        } else if (!StringUtils.endsWith(upper, BEGIN)) {

            carrier.car.db = qe.getDbName();
            for (String sql : query.split(";")) {
                SimpleDdlParser.DdlResult result = SimpleDdlParser.parse(sql.trim(), qe.getDbName());
                String schema = qe.getDbName();

                switch (result.getType()) {
                    case RENAME:
                        for (SimpleDdlParser.DdlResult rst : result.getRsts()) {
                            handleDDLParseResult(rst, schema, tableMetaCache, carrier);
                        }
                        break;
                    default:
                        handleDDLParseResult(result, schema, tableMetaCache, carrier);
                        break;
                }
            }
            return true; // ddl 语句也需要保存不能够直接从队列当中拿走 兼容gtid操作
        }

        return false;
    }

    /**
     * handle ddl result
     *
     * @param rst
     * @param schema
     */
    private final void handleDDLParseResult(SimpleDdlParser.DdlResult rst, String schema,
                                            TableMetaCache tableMetaCache, Carrier carrier) {
        if (StringUtils.isNotEmpty(rst.getSchemaName())) {
            schema = rst.getSchemaName();
        }
        String table = rst.getTableName();

        if (table == null || schema == null || rst.getType().equals(WaveEntry.EventType.ERASE)) {
            return;
        }
        // 如果新的表名是 _old 或者 _new形式 则直接放弃不需要table meta
        if (!ptOnlineTableFilter.filter(rst.getTableName())) {
            // 非begin commit 由于过滤规则需要 也需要将库表写入到carrier 载体当中
            carrier.car.table = rst.getTableName();

            tableMetaCache.refreshTableCache(schema, rst.getTableName());
        }
    }

    /**
     * 根据binlog事件当中列的数量 与 缓存当中列的数量是否一致
     * <p>
     * 决定是否需要刷新缓存的表结构信息
     *
     * @param columnInfo
     * @param db
     * @param table
     * @param tableMetaCache
     * @return TableMeta
     */
    private TableMeta updateTableMetaByCase(TableMapLogEvent.ColumnInfo[] columnInfo,
                                            String db, String table,
                                            TableMetaCache tableMetaCache) {
        if (db == null || table == null) {
            return null; // example like: CREATE DATABASE IF NOT EXITS BDAUTH
        }
        TableMeta tableMeta = tableMetaCache.getTableMeta(db, table, true);

        if (tableMeta == null) {
            tableMeta = tableMetaCache.refreshTableCache(db, table); // 如果表结构不存在 直接强制刷新
        }

        // check table fileds count,只能处理加字段
        if (tableMeta != null && columnInfo.length > tableMeta.getColumnSize()) {
            // online ddl增加字段操作步骤:
            // 1. 新增一张临时表,将需要做ddl表的数据全量导入
            // 2. 在老表上建立I/U/D的trigger,增量的将数据插入到临时表
            // 3. 锁住应用请求,将临时表rename为老表的名字,完成增加字段的操作
            // 尝试做一次reload,可能因为ddl没有正确解析,或者使用了类似online ddl的操作
            // 因为online ddl没有对应表名的alter语法,所以不会有clear cache的操作
            tableMeta = tableMetaCache.refreshTableCache(db, table);

            // 再做一次判断
            if (columnInfo.length > tableMeta.getColumnSize()) {
                LogUtils.warn.warn("column size is not match for table:"
                        + tableMeta.getFullName() + ","
                        + columnInfo.length + " vs " + tableMeta.getColumnSize());
            }
        }
        return tableMeta;
    }

    /**
     * add new gtid into gtids
     *
     * @param gtids
     * @param newGtid
     */
    private final void addNewGtid(Map<String, GTID> gtids, GTID newGtid) {
        String sid = newGtid.sid;
        if (gtids.containsKey(sid)) {
            GTID gtid = gtids.get(sid);
            newGtid.mergeGtid(gtid);
            gtids.put(sid, newGtid);
            return;
        }

        gtids.put(sid, newGtid);
    }

    private void prepare() throws BinlogException {
        LogUtils.debug.debug("prepare");
        this.originPos = getLogPos();
        this.dump.connect();
        updateConnectionProps(dump);

        if (originPos.hasGTID() && metaInfo.getBinlogInfo().getWithGTID()) {
            LogUtils.info.info(host + " binlog dump : GTID DUMP");

            BinlogDump.sendDumpCommand(dump, DumpType.COM_BINLOG_DUMP_GTID, metaInfo);

        } else {
            LogUtils.info.info(host + " binlog dump : COMMON DUMP");

            BinlogDump.sendDumpCommand(dump, DumpType.COM_BINLOG_DUMP, metaInfo);
        }

        // 过滤事件 而不是dump 全部事件
        decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);

        context = new LogContext();
        context.setLogPosition(
                new LogPosition(metaInfo.getBinlogFileName(), metaInfo.getBinlogPosition(),
                        metaInfo.getBinlogInfo().getBinlogWhen(), true));
    }

    /**
     * update MySQL connection properties
     *
     * @param connector
     * @throws IOException
     */
    protected void updateConnectionProps(MySQLConnector connector) {
        MySQLExecutor exe = new MySQLExecutor(connector);
        exe.setConnCfg(metaInfo.getDbInfo().getSlaveUUID());
    }

    /**
     * gtid 格式: 9583d493-ce62-11e6-91a1-507b9d578e91:1-4:6-9:12-15:18
     *
     * @return LoPostion get origin log position
     */
    private LogPosition getLogPos() {
        Meta.BinlogInfo binlogInfo = metaInfo.getBinlogInfo();
        LogPosition logPosition = new LogPosition(binlogInfo.getBinlogFile(), binlogInfo.getBinlogPos(),
                binlogInfo.getBinlogWhen(), true);

        if (binlogInfo.getExecutedGtidSets().length() > 0) {
            for (String gtidSet : binlogInfo.getExecutedGtidSets().split(",")) {
                logPosition.addGtid(GTID.parseGTID(gtidSet.split(":")));
            }
            logPosition.refresh();
        }
        LogUtils.info.info("origin pos : " + logPosition.getGtidSets());
        return logPosition;
    }
}