com.taobao.tddl.dbsync.binlog.LogEvent Java Examples

The following examples show how to use com.taobao.tddl.dbsync.binlog.LogEvent. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TableMapLogEvent.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
private void parse_geometry_type(LogBuffer buffer, int length) {
    // stores geometry column's types extracted from field.
    int limit = buffer.position() + length;

    List<Integer> datas = new ArrayList<Integer>();
    while (buffer.hasRemaining() && buffer.position() < limit) {
        int col_type = (int) buffer.getPackedLong();
        datas.add(col_type);
    }

    int index = 0;
    for (int i = 0; i < columnCnt; i++) {
        if (columnInfo[i].type == LogEvent.MYSQL_TYPE_GEOMETRY) {
            columnInfo[i].geoType = datas.get(index);
            index++;
        }
    }
}
Example #2
Source File: MysqlMultiStageCoprocessor.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
@Override
public void onEvent(MessageEvent event) throws Exception {
    try {
        if (event.isNeedDmlParse()) {
            int eventType = event.getEvent().getHeader().getType();
            CanalEntry.Entry entry = null;
            switch (eventType) {
                case LogEvent.ROWS_QUERY_LOG_EVENT:
                    entry = logEventConvert.parse(event.getEvent(), false);
                    break;
                default:
                    // 单独解析dml事件
                    entry = logEventConvert.parseRowsEvent((RowsLogEvent) event.getEvent(), event.getTable());
            }

            event.setEntry(entry);
        }
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}
Example #3
Source File: TableMapLogEvent.java    From canal with Apache License 2.0 6 votes vote down vote up 
private void parse_geometry_type(LogBuffer buffer, int length) {
    // stores geometry column's types extracted from field.
    int limit = buffer.position() + length;

    List<Integer> datas = new ArrayList<Integer>();
    while (buffer.hasRemaining() && buffer.position() < limit) {
        int col_type = (int) buffer.getPackedLong();
        datas.add(col_type);
    }

    int index = 0;
    for (int i = 0; i < columnCnt; i++) {
        if (columnInfo[i].type == LogEvent.MYSQL_TYPE_GEOMETRY) {
            columnInfo[i].geoType = datas.get(index);
            index++;
        }
    }
}
Example #4
Source File: TableMapLogEvent.java    From canal with Apache License 2.0 6 votes vote down vote up 
private int getRealType(int type, int meta) {
    if (type == LogEvent.MYSQL_TYPE_STRING) {
        if (meta >= 256) {
            int byte0 = meta >> 8;
            if ((byte0 & 0x30) != 0x30) {
                /* a long CHAR() field: see #37426 */
                type = byte0 | 0x30;
            } else {
                switch (byte0) {
                    case LogEvent.MYSQL_TYPE_SET:
                    case LogEvent.MYSQL_TYPE_ENUM:
                    case LogEvent.MYSQL_TYPE_STRING:
                        type = byte0;
                }
            }
        }
    }

    return type;
}
Example #5
Source File: RowsLogEvent.java    From canal with Apache License 2.0 6 votes vote down vote up 
public final void fillTable(LogContext context) {
    table = context.getTable(tableId);

    if (table == null) {
        throw new TableIdNotFoundException("not found tableId:" + tableId);
    }

    // end of statement check:
    if ((flags & RowsLogEvent.STMT_END_F) != 0) {
        // Now is safe to clear ignored map (clear_tables will also
        // delete original table map events stored in the map).
        context.clearAllTables();
    }

    int jsonColumnCount = 0;
    int columnCnt = table.getColumnCnt();
    ColumnInfo[] columnInfo = table.getColumnInfo();
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        if (info.type == LogEvent.MYSQL_TYPE_JSON) {
            jsonColumnCount++;
        }
    }
    this.jsonColumnCount = jsonColumnCount;
}
Example #6
Source File: MysqlMultiStageCoprocessor.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event.getEntry() != null) {
            transactionBuffer.add(event.getEntry());
        }

        LogEvent logEvent = event.getEvent();
        if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
            // semi ack回报
            ((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
                logEvent.getHeader().getLogPos());
        }

        // clear for gc
        event.setBuffer(null);
        event.setEvent(null);
        event.setTable(null);
        event.setEntry(null);
        event.setNeedDmlParse(false);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}
Example #7
Source File: MysqlBinlogParsePerformanceTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
    try {
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                parseOneRow(event, buffer, columns, false);
            } else {
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                parseOneRow(event, buffer, changeColumns, true);
            }

            sum.incrementAndGet();
        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}
Example #8
Source File: MysqlConnection.java    From canal with Apache License 2.0 6 votes vote down vote up 
/**
 * 获取主库checksum信息
 * 
 * <pre>
 * mariadb区别于mysql会在binlog的第一个事件Rotate_Event里也会采用checksum逻辑,而mysql是在第二个binlog事件之后才感知是否需要处理checksum
 * 导致maraidb只要是开启checksum就会出现binlog文件名解析乱码
 * fixed issue : https://github.com/alibaba/canal/issues/1081
 * </pre>
 */
private void loadBinlogChecksum() {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum");
        List<String> columnValues = rs.getFieldValues();
        if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0) != null
            && columnValues.get(0).toUpperCase().equals("CRC32")) {
            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
        } else {
            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
        }
    } catch (Throwable e) {
        // logger.error("", e);
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}
Example #9
Source File: MysqlConnection.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
/**
 * 获取主库checksum信息
 * 
 * <pre>
 * mariadb区别于mysql会在binlog的第一个事件Rotate_Event里也会采用checksum逻辑,而mysql是在第二个binlog事件之后才感知是否需要处理checksum
 * 导致maraidb只要是开启checksum就会出现binlog文件名解析乱码
 * fixed issue : https://github.com/alibaba/canal/issues/1081
 * </pre>
 */
private void loadBinlogChecksum() {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum");
        List<String> columnValues = rs.getFieldValues();
        if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
        } else {
            binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
        }
    } catch (Throwable e) {
        logger.error("", e);
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}
Example #10
Source File: MysqlMultiStageCoprocessor.java    From canal with Apache License 2.0 6 votes vote down vote up 
@Override
public void onEvent(MessageEvent event) throws Exception {
    try {
        if (event.isNeedDmlParse()) {
            int eventType = event.getEvent().getHeader().getType();
            CanalEntry.Entry entry = null;
            switch (eventType) {
                case LogEvent.ROWS_QUERY_LOG_EVENT:
                    entry = logEventConvert.parse(event.getEvent(), false);
                    break;
                default:
                    // 单独解析dml事件
                    entry = logEventConvert.parseRowsEvent((RowsLogEvent) event.getEvent(), event.getTable());
            }

            event.setEntry(entry);
        }
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}
Example #11
Source File: MysqlMultiStageCoprocessor.java    From canal with Apache License 2.0 6 votes vote down vote up 
public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event.getEntry() != null) {
            transactionBuffer.add(event.getEntry());
        }

        LogEvent logEvent = event.getEvent();
        if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
            // semi ack回报
            ((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
                logEvent.getHeader().getLogPos());
        }

        // clear for gc
        event.setBuffer(null);
        event.setEvent(null);
        event.setTable(null);
        event.setEntry(null);
        event.setNeedDmlParse(false);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}
Example #12
Source File: MysqlBinlogParsePerformanceTest.java    From canal with Apache License 2.0 6 votes vote down vote up 
public static void parseRowsEvent(RowsLogEvent event, AtomicLong sum) {
    try {
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                parseOneRow(event, buffer, columns, false);
            } else {
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                parseOneRow(event, buffer, changeColumns, true);
            }

            sum.incrementAndGet();
        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}
Example #13
Source File: RowsLogEvent.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
public final void fillTable(LogContext context) {
    table = context.getTable(tableId);

    // end of statement check:
    if ((flags & RowsLogEvent.STMT_END_F) != 0) {
        // Now is safe to clear ignored map (clear_tables will also
        // delete original table map events stored in the map).
        context.clearAllTables();
    }

    int jsonColumnCount = 0;
    int columnCnt = table.getColumnCnt();
    ColumnInfo[] columnInfo = table.getColumnInfo();
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        if (info.type == LogEvent.MYSQL_TYPE_JSON) {
            jsonColumnCount++;
        }
    }
    this.jsonColumnCount = jsonColumnCount;
}
Example #14
Source File: TableMapLogEvent.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up 
private int getRealType(int type, int meta) {
    if (type == LogEvent.MYSQL_TYPE_STRING) {
        if (meta >= 256) {
            int byte0 = meta >> 8;
            if ((byte0 & 0x30) != 0x30) {
                /* a long CHAR() field: see #37426 */
                type = byte0 | 0x30;
            } else {
                switch (byte0) {
                    case LogEvent.MYSQL_TYPE_SET:
                    case LogEvent.MYSQL_TYPE_ENUM:
                    case LogEvent.MYSQL_TYPE_STRING:
                        type = byte0;
                }
            }
        }
    }

    return type;
}
Example #15
Source File: MysqlConnection.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
@Override
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDumpGTID(gtidSet);

    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    try {
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        // fix bug: #890 将gtid传输至context中，供decode使用
        context.setGtidSet(gtidSet);
        while (fetcher.fetch()) {
            accumulateReceivedBytes(fetcher.limit());
            LogEvent event = null;
            event = decoder.decode(fetcher, context);

            if (event == null) {
                throw new CanalParseException("parse failed");
            }

            if (!func.sink(event)) {
                break;
            }
        }
    } finally {
        fetcher.close();
    }
}
Example #16
Source File: MysqlMultiStageCoprocessor.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public SimpleParserStage(LogContext context){
    decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    this.context = context;
    if (gtidSet != null) {
        context.setGtidSet(gtidSet);
    }
}
Example #17
Source File: GroupEventPaserTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private BinlogParser buildParser(AuthenticationInfo info) {
    return new AbstractBinlogParser<LogEvent>() {

        @Override
        public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException {
            return null;
        }
    };
}
Example #18
Source File: MysqlConnection.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendRegisterSlave();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);

        if (event == null) {
            throw new CanalParseException("parse failed");
        }

        if (!func.sink(event)) {
            break;
        }

        if (event.getSemival() == 1) {
            sendSemiAck(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
        }
    }
}
Example #19
Source File: DirectLogFetcherTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected void parseRowsEvent(RowsLogEvent event) {
    try {
        System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
            binlogFileName,
            event.getHeader().getLogPos() - event.getHeader().getEventLen(),
            event.getTable().getDbName(),
            event.getTable().getTableName()));
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                // insert的记录放在before字段中
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                // delete的记录放在before字段中
                parseOneRow(event, buffer, columns, false);
            } else {
                // update需要处理before/after
                System.out.println("-------> before");
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                System.out.println("-------> after");
                parseOneRow(event, buffer, changeColumns, true);
            }

        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}
Example #20
Source File: MysqlConnection.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
/**
 * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
 */
public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.ROTATE_EVENT);
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    // 若entry position存在gtid，则使用传入的gtid作为gtidSet
    // 拼接的标准,否则同时开启gtid和tsdb时，会导致丢失gtid
    // 而当源端数据库gtid 有purged时会有如下类似报错
    // 'errno = 1236, sqlstate = HY000 errmsg = The slave is connecting
    // using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
    if (StringUtils.isNotEmpty(gtid)) {
        decoder.handle(LogEvent.GTID_LOG_EVENT);
        context.setGtidSet(MysqlGTIDSet.parse(gtid));
    }
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);

        if (event == null) {
            throw new CanalParseException("parse failed");
        }

        if (!func.sink(event)) {
            break;
        }
    }
}
Example #21
Source File: DirectLogFetcherTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private void loadBinlogChecksum(MysqlConnector connector) {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum", connector);
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}
Example #22
Source File: DirectLogFetcherTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
protected void parseRowsEvent(RowsLogEvent event) {
    try {
        System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s]",
            binlogFileName,
            event.getHeader().getLogPos() - event.getHeader().getEventLen(),
            event.getTable().getDbName(),
            event.getTable().getTableName()));
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            int type = event.getHeader().getType();
            if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
                // insert的记录放在before字段中
                parseOneRow(event, buffer, columns, true);
            } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
                // delete的记录放在before字段中
                parseOneRow(event, buffer, columns, false);
            } else {
                // update需要处理before/after
                System.out.println("-------> before");
                parseOneRow(event, buffer, columns, false);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    break;
                }
                System.out.println("-------> after");
                parseOneRow(event, buffer, changeColumns, true);
            }

        }
    } catch (Exception e) {
        throw new RuntimeException("parse row data failed.", e);
    }
}
Example #23
Source File: TableMapLogEvent.java    From canal with Apache License 2.0 5 votes vote down vote up 
private void parse_set_str_value(LogBuffer buffer, int length, boolean set) {
    // stores SET/ENUM column's string values extracted from
    // field. Each SET/ENUM column's string values are stored
    // into a string separate vector. All of them are stored
    // in 'vec'.
    int limit = buffer.position() + length;
    List<List<String>> datas = new ArrayList<List<String>>();
    while (buffer.hasRemaining() && buffer.position() < limit) {
        int count = (int) buffer.getPackedLong();
        List<String> data = new ArrayList<String>(count);
        for (int i = 0; i < count; i++) {
            int len1 = (int) buffer.getPackedLong();
            data.add(buffer.getFixString(len1));
        }

        datas.add(data);
    }

    int index = 0;
    for (int i = 0; i < columnCnt; i++) {
        if (set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_SET) {
            columnInfo[i].set_enum_values = datas.get(index);
            index++;
        }

        if (!set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_ENUM) {
            columnInfo[i].set_enum_values = datas.get(index);
            index++;
        }
    }
}
Example #24
Source File: MysqlConnection.java    From canal with Apache License 2.0 5 votes vote down vote up 
/**
 * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
 */
public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.ROTATE_EVENT);
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    // 若entry position存在gtid，则使用传入的gtid作为gtidSet
    // 拼接的标准,否则同时开启gtid和tsdb时，会导致丢失gtid
    // 而当源端数据库gtid 有purged时会有如下类似报错
    // 'errno = 1236, sqlstate = HY000 errmsg = The slave is connecting
    // using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
    if (StringUtils.isNotEmpty(gtid)) {
        decoder.handle(LogEvent.GTID_LOG_EVENT);
        context.setGtidSet(MysqlGTIDSet.parse(gtid));
    }
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);

        if (event == null) {
            throw new CanalParseException("parse failed");
        }

        if (!func.sink(event)) {
            break;
        }
    }
}
Example #25
Source File: MysqlConnection.java    From canal with Apache License 2.0 5 votes vote down vote up 
public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendRegisterSlave();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);

        if (event == null) {
            throw new CanalParseException("parse failed");
        }

        if (!func.sink(event)) {
            break;
        }

        if (event.getSemival() == 1) {
            sendSemiAck(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
        }
    }
}
Example #26
Source File: MysqlConnection.java    From canal with Apache License 2.0 5 votes vote down vote up 
@Override
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDumpGTID(gtidSet);

    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    try {
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        // fix bug: #890 将gtid传输至context中，供decode使用
        context.setGtidSet(gtidSet);
        while (fetcher.fetch()) {
            accumulateReceivedBytes(fetcher.limit());
            LogEvent event = null;
            event = decoder.decode(fetcher, context);

            if (event == null) {
                throw new CanalParseException("parse failed");
            }

            if (!func.sink(event)) {
                break;
            }
        }
    } finally {
        fetcher.close();
    }
}
Example #27
Source File: MysqlMultiStageCoprocessor.java    From canal with Apache License 2.0 5 votes vote down vote up 
public SimpleParserStage(LogContext context){
    decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    this.context = context;
    if (gtidSet != null) {
        context.setGtidSet(gtidSet);
    }
}
Example #28
Source File: GroupEventPaserTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private BinlogParser buildParser(AuthenticationInfo info) {
    return new AbstractBinlogParser<LogEvent>() {

        @Override
        public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException {
            return null;
        }
    };
}
Example #29
Source File: DirectLogFetcherTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private void loadBinlogChecksum(MysqlConnector connector) {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum", connector);
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}
Example #30
Source File: TableMapLogEvent.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private void parse_set_str_value(LogBuffer buffer, int length, boolean set) {
    // stores SET/ENUM column's string values extracted from
    // field. Each SET/ENUM column's string values are stored
    // into a string separate vector. All of them are stored
    // in 'vec'.
    int limit = buffer.position() + length;
    List<List<String>> datas = new ArrayList<List<String>>();
    while (buffer.hasRemaining() && buffer.position() < limit) {
        int count = (int) buffer.getPackedLong();
        List<String> data = new ArrayList<String>(count);
        for (int i = 0; i < count; i++) {
            int len1 = (int) buffer.getPackedLong();
            data.add(buffer.getFixString(len1));
        }

        datas.add(data);
    }

    int index = 0;
    for (int i = 0; i < columnCnt; i++) {
        if (set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_SET) {
            columnInfo[i].set_enum_values = datas.get(index);
            index++;
        }

        if (!set && getRealType(columnInfo[i].type, columnInfo[i].meta) == LogEvent.MYSQL_TYPE_ENUM) {
            columnInfo[i].set_enum_values = datas.get(index);
            index++;
        }
    }
}