Java Code Examples for com.taobao.tddl.dbsync.binlog.LogEvent#UNKNOWN_EVENT

The following examples show how to use com.taobao.tddl.dbsync.binlog.LogEvent#UNKNOWN_EVENT . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MysqlConnection.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendRegisterSlave();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);

        if (event == null) {
            throw new CanalParseException("parse failed");
        }

        if (!func.sink(event)) {
            break;
        }

        if (event.getSemival() == 1) {
            sendSemiAck(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
        }
    }
}
Example 2
Source File: MysqlConnection.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
@Override
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDumpGTID(gtidSet);

    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    try {
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        // fix bug: #890 将gtid传输至context中，供decode使用
        context.setGtidSet(gtidSet);
        while (fetcher.fetch()) {
            accumulateReceivedBytes(fetcher.limit());
            LogEvent event = null;
            event = decoder.decode(fetcher, context);

            if (event == null) {
                throw new CanalParseException("parse failed");
            }

            if (!func.sink(event)) {
                break;
            }
        }
    } finally {
        fetcher.close();
    }
}
Example 3
Source File: MysqlMultiStageCoprocessor.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public SimpleParserStage(LogContext context){
    decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    this.context = context;
    if (gtidSet != null) {
        context.setGtidSet(gtidSet);
    }
}
Example 4
Source File: MysqlConnection.java    From canal with Apache License 2.0 5 votes vote down vote up 
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendRegisterSlave();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);

        if (event == null) {
            throw new CanalParseException("parse failed");
        }

        if (!func.sink(event)) {
            break;
        }

        if (event.getSemival() == 1) {
            sendSemiAck(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
        }
    }
}
Example 5
Source File: MysqlConnection.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Override
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDumpGTID(gtidSet);

    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    try {
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        // fix bug: #890 将gtid传输至context中，供decode使用
        context.setGtidSet(gtidSet);
        while (fetcher.fetch()) {
            accumulateReceivedBytes(fetcher.limit());
            LogEvent event = null;
            event = decoder.decode(fetcher, context);

            if (event == null) {
                throw new CanalParseException("parse failed");
            }

            if (!func.sink(event)) {
                break;
            }
        }
    } finally {
        fetcher.close();
    }
}
Example 6
Source File: MysqlMultiStageCoprocessor.java    From canal with Apache License 2.0 5 votes vote down vote up 
public SimpleParserStage(LogContext context){
    decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    this.context = context;
    if (gtidSet != null) {
        context.setGtidSet(gtidSet);
    }
}
Example 7
Source File: LocalBinLogConnection.java    From canal-1.1.3 with Apache License 2.0 4 votes vote down vote up 
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    File current = new File(directory, binlogfilename);

    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                if (serverId != 0 && event.getServerId() != serverId) {
                    throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
                }

                if (!func.sink(event)) {
                    needContinue = false;
                    break;
                }
            }

            fetcher.close(); // 关闭上一个文件
            parserFinish(current.getName());
            if (needContinue) {// 读取下一个

                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }

                if (nextFile == null) {
                    break;
                }

                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(nextFile.getName()));
            } else {
                break;// 跳出
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
}
Example 8
Source File: LocalBinLogConnection.java    From canal-1.1.3 with Apache License 2.0 4 votes vote down vote up 
@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
    File current = new File(directory, binlogfilename);
    if (!current.exists()) {
        throw new CanalParseException("binlog:" + binlogfilename + " is not found");
    }

    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                if (serverId != 0 && event.getServerId() != serverId) {
                    throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
                }

                if (!coprocessor.publish(event)) {
                    needContinue = false;
                    break;
                }
            }

            fetcher.close(); // 关闭上一个文件
            parserFinish(binlogfilename);
            if (needContinue) {// 读取下一个
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }

                if (nextFile == null) {
                    break;
                }

                current = nextFile;
                fetcher.open(current);
                binlogfilename = nextFile.getName();
            } else {
                break;// 跳出
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
}
Example 9
Source File: MysqlBinlogParsePerformanceTest.java    From canal-1.1.3 with Apache License 2.0 4 votes vote down vote up 
public static void consumer(BlockingQueue<LogBuffer> buffer) throws IOException, InterruptedException {
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();

    AtomicLong sum = new AtomicLong(0);
    long start = System.currentTimeMillis();
    long last = 0;
    long end = 0;
    while (true) {
        LogEvent event = null;
        event = decoder.decode(buffer.take(), context);
        int eventType = event.getHeader().getType();
        switch (eventType) {
            case LogEvent.ROTATE_EVENT:
                break;
            case LogEvent.WRITE_ROWS_EVENT_V1:
            case LogEvent.WRITE_ROWS_EVENT:
                parseRowsEvent((WriteRowsLogEvent) event, sum);
                break;
            case LogEvent.UPDATE_ROWS_EVENT_V1:
            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
            case LogEvent.UPDATE_ROWS_EVENT:
                parseRowsEvent((UpdateRowsLogEvent) event, sum);
                break;
            case LogEvent.DELETE_ROWS_EVENT_V1:
            case LogEvent.DELETE_ROWS_EVENT:
                parseRowsEvent((DeleteRowsLogEvent) event, sum);
                break;
            case LogEvent.XID_EVENT:
                sum.incrementAndGet();
                break;
            case LogEvent.QUERY_EVENT:
                sum.incrementAndGet();
                break;
            default:
                break;
        }

        long current = sum.get();
        if (current - last >= 100000) {
            end = System.currentTimeMillis();
            long tps = ((current - last) * 1000) / (end - start);
            System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
            last = current;
            start = end;
        }
    }
}
Example 10
Source File: LocalBinLogConnection.java    From canal with Apache License 2.0 4 votes vote down vote up 
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    File current = new File(directory, binlogfilename);

    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                if (serverId != 0 && event.getServerId() != serverId) {
                    throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
                }

                if (!func.sink(event)) {
                    needContinue = false;
                    break;
                }
            }

            fetcher.close(); // 关闭上一个文件
            parserFinish(current.getName());
            if (needContinue) {// 读取下一个

                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }

                if (nextFile == null) {
                    break;
                }

                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(nextFile.getName()));
            } else {
                break;// 跳出
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
}
Example 11
Source File: LocalBinLogConnection.java    From canal with Apache License 2.0 4 votes vote down vote up 
@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
    File current = new File(directory, binlogfilename);
    if (!current.exists()) {
        throw new CanalParseException("binlog:" + binlogfilename + " is not found");
    }

    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                if (serverId != 0 && event.getServerId() != serverId) {
                    throw new ServerIdNotMatchException("unexpected serverId " + serverId + " in binlog file !");
                }

                if (!coprocessor.publish(event)) {
                    needContinue = false;
                    break;
                }
            }

            fetcher.close(); // 关闭上一个文件
            parserFinish(binlogfilename);
            if (needContinue) {// 读取下一个
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }

                if (nextFile == null) {
                    break;
                }

                current = nextFile;
                fetcher.open(current);
                binlogfilename = nextFile.getName();
            } else {
                break;// 跳出
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
}
Example 12
Source File: MysqlBinlogParsePerformanceTest.java    From canal with Apache License 2.0 4 votes vote down vote up 
public static void consumer(BlockingQueue<LogBuffer> buffer) throws IOException, InterruptedException {
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();

    AtomicLong sum = new AtomicLong(0);
    long start = System.currentTimeMillis();
    long last = 0;
    long end = 0;
    while (true) {
        LogEvent event = null;
        event = decoder.decode(buffer.take(), context);
        int eventType = event.getHeader().getType();
        switch (eventType) {
            case LogEvent.ROTATE_EVENT:
                break;
            case LogEvent.WRITE_ROWS_EVENT_V1:
            case LogEvent.WRITE_ROWS_EVENT:
                parseRowsEvent((WriteRowsLogEvent) event, sum);
                break;
            case LogEvent.UPDATE_ROWS_EVENT_V1:
            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
            case LogEvent.UPDATE_ROWS_EVENT:
                parseRowsEvent((UpdateRowsLogEvent) event, sum);
                break;
            case LogEvent.DELETE_ROWS_EVENT_V1:
            case LogEvent.DELETE_ROWS_EVENT:
                parseRowsEvent((DeleteRowsLogEvent) event, sum);
                break;
            case LogEvent.XID_EVENT:
                sum.incrementAndGet();
                break;
            case LogEvent.QUERY_EVENT:
                sum.incrementAndGet();
                break;
            default:
                break;
        }

        long current = sum.get();
        if (current - last >= 100000) {
            end = System.currentTimeMillis();
            long tps = ((current - last) * 1000) / (end - start);
            System.out.println(" total : " + sum + " , cost : " + (end - start) + " , tps : " + tps);
            last = current;
            start = end;
        }
    }
}