com.alibaba.otter.canal.parse.exception.CanalParseException Java Examples

The following examples show how to use com.alibaba.otter.canal.parse.exception.CanalParseException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AbstractEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}

Example #2

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

protected void afterDump(ErosaConnection connection) {
    super.afterDump(connection);

    if (connection == null) {
        throw new CanalParseException("illegal connection is null");
    }

    if (!(connection instanceof MysqlConnection)) {
        throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
    }

    if (metaConnection != null) {
        try {
            metaConnection.disconnect();
        } catch (IOException e) {
            logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
                .getAddress(), e);
        }
    }
}

Example #3

Source File: MysqlConnection.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 获取一下binlog format格式
 */
private void loadBinlogFormat() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_format'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        logger.warn("unexpected binlog format query result, this may cause unexpected result, so throw exception to request network to io shutdown.");
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }

    binlogFormat = BinlogFormat.valuesOf(columnValues.get(1));
    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }
}

Example #4

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        String tableName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, null);
            if (results.size() > 0) {
                tableName = results.get(0).getTableName();
            }
        }

        return buildQueryEntry(queryString, event.getHeader(), tableName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}

Example #5

Source File: LocalBinlogEventParser.java From canal with Apache License 2.0

6 votes

@Override
protected void preDump(ErosaConnection connection) {
    metaConnection = buildMysqlConnection();
    try {
        metaConnection.connect();
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
        ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
        ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
        ((DatabaseTableMeta) tableMetaTSDB).init(destination);
    }

    tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
    ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}

Example #6

Source File: MysqlConnection.java From canal with Apache License 2.0

6 votes

/**
 * 获取一下binlog image格式
 */
private void loadBinlogImage() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_row_image'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        // 可能历时版本没有image特性
        binlogImage = BinlogImage.FULL;
    } else {
        binlogImage = BinlogImage.valuesOf(columnValues.get(1));
    }

    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
    }
}

Example #7

Source File: LocalBinlogEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
protected void preDump(ErosaConnection connection) {
    metaConnection = buildMysqlConnection();
    try {
        metaConnection.connect();
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
        ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
        ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
        ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
        ((DatabaseTableMeta) tableMetaTSDB).init(destination);
    }

    tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
    ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
}

Example #8

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

6 votes

public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event.getEntry() != null) {
            transactionBuffer.add(event.getEntry());
        }

        LogEvent logEvent = event.getEvent();
        if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
            // semi ack回报
            ((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
                logEvent.getHeader().getLogPos());
        }

        // clear for gc
        event.setBuffer(null);
        event.setEvent(null);
        event.setTable(null);
        event.setEntry(null);
        event.setNeedDmlParse(false);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}

Example #9

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

private final long generateUniqueServerId() {
    try {
        // a=`echo $masterip|cut -d\. -f1`
        // b=`echo $masterip|cut -d\. -f2`
        // c=`echo $masterip|cut -d\. -f3`
        // d=`echo $masterip|cut -d\. -f4`
        // #server_id=`expr $a \* 256 \* 256 \* 256 + $b \* 256 \* 256 + $c
        // \* 256 + $d `
        // #server_id=$b$c$d
        // server_id=`expr $b \* 256 \* 256 + $c \* 256 + $d `
        InetAddress localHost = InetAddress.getLocalHost();
        byte[] addr = localHost.getAddress();
        int salt = (destination != null) ? destination.hashCode() : 0;
        return ((0x7f & salt) << 24) + ((0xff & (int) addr[1]) << 16) // NL
               + ((0xff & (int) addr[2]) << 8) // NL
               + (0xff & (int) addr[3]);
    } catch (UnknownHostException e) {
        throw new CanalParseException("Unknown host", e);
    }
}

Example #10

Source File: MysqlConnection.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 获取一下binlog image格式
 */
private void loadBinlogImage() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_row_image'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        // 可能历时版本没有image特性
        binlogImage = BinlogImage.FULL;
    } else {
        binlogImage = BinlogImage.valuesOf(columnValues.get(1));
    }

    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
    }
}

Example #11

Source File: LogEventConvert.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #12

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        String tableName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, null);
            if (results.size() > 0) {
                tableName = results.get(0).getTableName();
            }
        }

        return buildQueryEntry(queryString, event.getHeader(), tableName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}

Example #13

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #14

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected void afterDump(ErosaConnection connection) {
    super.afterDump(connection);

    if (connection == null) {
        throw new CanalParseException("illegal connection is null");
    }

    if (!(connection instanceof MysqlConnection)) {
        throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
    }

    if (metaConnection != null) {
        try {
            metaConnection.disconnect();
        } catch (IOException e) {
            logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
                .getAddress(), e);
        }
    }
}

Example #15

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// �Ƚ�before / after��column index
            if (column.getIsNull() && newValue == null) {
                // ���ȫ��null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // �����Ϊnull���������
                return false;
            }
        }
    }

    // ����nolob/minialģʽ��,�����Ҳ���before��¼,��Ϊ���б仯
    return true;
}

Example #16

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

private final long generateUniqueServerId() {
    try {
        // a=`echo $masterip|cut -d\. -f1`
        // b=`echo $masterip|cut -d\. -f2`
        // c=`echo $masterip|cut -d\. -f3`
        // d=`echo $masterip|cut -d\. -f4`
        // #server_id=`expr $a \* 256 \* 256 \* 256 + $b \* 256 \* 256 + $c
        // \* 256 + $d `
        // #server_id=$b$c$d
        // server_id=`expr $b \* 256 \* 256 + $c \* 256 + $d `
        InetAddress localHost = InetAddress.getLocalHost();
        byte[] addr = localHost.getAddress();
        int salt = (destination != null) ? destination.hashCode() : 0;
        return ((0x7f & salt) << 24) + ((0xff & (int) addr[1]) << 16) // NL
               + ((0xff & (int) addr[2]) << 8) // NL
               + (0xff & (int) addr[3]);
    } catch (UnknownHostException e) {
        throw new CanalParseException("Unknown host", e);
    }
}

Example #17

Source File: LogEventConvert.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #18

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 查询当前的binlog位置
 */
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show master status");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        if (isGTIDMode() && fields.size() > 4) {
            endPosition.setGtid(fields.get(4));
        }
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show master status' has an error!", e);
    }
}

Example #19

Source File: AbstractEventParser.java From DBus with Apache License 2.0

6 votes

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}

Example #20

Source File: MysqlMultiStageCoprocessor.java From canal-1.1.3 with Apache License 2.0

6 votes

public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        if (event.getEntry() != null) {
            transactionBuffer.add(event.getEntry());
        }

        LogEvent logEvent = event.getEvent();
        if (connection instanceof MysqlConnection && logEvent.getSemival() == 1) {
            // semi ack回报
            ((MysqlConnection) connection).sendSemiAck(logEvent.getHeader().getLogFileName(),
                logEvent.getHeader().getLogPos());
        }

        // clear for gc
        event.setBuffer(null);
        event.setEvent(null);
        event.setTable(null);
        event.setEntry(null);
        event.setNeedDmlParse(false);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}

Example #21

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #22

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #23

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

public void start() throws CanalParseException {
    if (enableTsdb) {
        if (tableMetaTSDB == null) {
            synchronized (CanalEventParser.class) {
                try {
                    // 设置当前正在加载的通道，加载spring查找文件时会用到该变量
                    System.setProperty("canal.instance.destination", destination);
                    // 初始化
                    tableMetaTSDB = tableMetaTSDBFactory.build(destination, tsdbSpringXml);
                } finally {
                    System.setProperty("canal.instance.destination", "");
                }
            }
        }
    }

    super.start();
}

Example #24

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

public void stop() throws CanalParseException {
    if (metaConnection != null) {
        try {
            metaConnection.disconnect();
        } catch (IOException e) {
            logger.error("ERROR # disconnect meta connection for address:{}", metaConnection.getConnector()
                .getAddress(), e);
        }
    }

    if (tableMetaCache != null) {
        tableMetaCache.clearTableMeta();
    }

    super.stop();
}

Example #25

Source File: FailbackLogPositionManager.java From canal with Apache License 2.0

5 votes

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    try {
        primary.persistLogPosition(destination, logPosition);
    } catch (CanalParseException e) {
        logger.warn("persistLogPosition use primary log position manager exception. destination: {}, logPosition: {}",
            destination,
            logPosition,
            e);
        secondary.persistLogPosition(destination, logPosition);
    }
}

Example #26

Source File: MixedLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
    memoryLogPositionManager.persistLogPosition(destination, logPosition);
    executor.submit(new Runnable() {

        public void run() {
            try {
                zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
            } catch (Exception e) {
                logger.error("ERROR # persist to zookeeper has an error", e);
            }
        }
    });
}

Example #27

Source File: DirectLogFetcherTest.java From canal with Apache License 2.0

5 votes

private void loadBinlogChecksum(MysqlConnector connector) {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum", connector);
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}

Example #28

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

5 votes

private EntryPosition buildMemFromSnapshot(EntryPosition position) {
    try {
        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp());
        if (snapshotDO == null) {
            return null;
        }
        String binlogFile = snapshotDO.getBinlogFile();
        Long binlogOffest = snapshotDO.getBinlogOffest();
        String binlogMasterId = snapshotDO.getBinlogMasterId();
        Long binlogTimestamp = snapshotDO.getBinlogTimestamp();

        EntryPosition snapshotPosition = new EntryPosition(binlogFile,
            binlogOffest == null ? 0l : binlogOffest,
            binlogTimestamp == null ? 0l : binlogTimestamp,
            Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
        // data存储为Map<String,String>，每个分库一套建表
        String sqlData = snapshotDO.getData();
        JSONObject jsonObj = JSON.parseObject(sqlData);
        for (Map.Entry entry : jsonObj.entrySet()) {
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition,
                ObjectUtils.toString(entry.getKey()),
                ObjectUtils.toString(entry.getValue()),
                null)) {
                return null;
            }
        }

        return snapshotPosition;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
    }
}

Example #29

Source File: DirectLogFetcherTest.java From canal-1.1.3 with Apache License 2.0

5 votes

private void loadBinlogChecksum(MysqlConnector connector) {
    ResultSetPacket rs = null;
    try {
        rs = query("select @@global.binlog_checksum", connector);
    } catch (IOException e) {
        throw new CanalParseException(e);
    }

    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}

Example #30

Source File: FailbackLogPositionManager.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    try {
        primary.persistLogPosition(destination, logPosition);
    } catch (CanalParseException e) {
        logger.warn("persistLogPosition use primary log position manager exception. destination: {}, logPosition: {}",
            destination,
            logPosition,
            e);
        secondary.persistLogPosition(destination, logPosition);
    }
}