Java Code Examples for com.alibaba.otter.canal.protocol.position.EntryPosition#getTimestamp()

The following examples show how to use com.alibaba.otter.canal.protocol.position.EntryPosition#getTimestamp() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) {
        // 使用一个未来极大的时间，基于位点进行定位
        long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年
        EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
            startTimestamp,
            fixedPosition,
            fixedPosition.getJournalName(),
            true);
        if (entryPosition == null) {
            throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position"
                                          + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition());
        }
        return entryPosition;
    } else {
        return fixedPosition;
    }
}

Example 2

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) {
        // 使用一个未来极大的时间，基于位点进行定位
        long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年
        EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
            startTimestamp,
            fixedPosition,
            fixedPosition.getJournalName(),
            true);
        if (entryPosition == null) {
            throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position"
                                          + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition());
        }
        return entryPosition;
    } else {
        return fixedPosition;
    }
}

Example 3

Source File: LocalBinlogEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}

Example 4

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

/**
 * 回滚到指定位点
 * 
 * @param position
 * @return
 */
protected boolean processTableMeta(EntryPosition position) {
    if (tableMetaTSDB != null) {
        if (position.getTimestamp() == null || position.getTimestamp() <= 0) {
            throw new CanalParseException("use gtid and TableMeta TSDB should be config timestamp > 0");
        }

        return tableMetaTSDB.rollback(position);
    }

    return true;
}

Example 5

Source File: LocalBinlogEventParser.java From canal with Apache License 2.0

5 votes

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}

Example 6

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

5 votes

/**
 * 回滚到指定位点
 * 
 * @param position
 * @return
 */
protected boolean processTableMeta(EntryPosition position) {
    if (tableMetaTSDB != null) {
        if (position.getTimestamp() == null || position.getTimestamp() <= 0) {
            throw new CanalParseException("use gtid and TableMeta TSDB should be config timestamp > 0");
        }

        return tableMetaTSDB.rollback(position);
    }

    return true;
}

Example 7

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean applyHistoryOnMemory(EntryPosition position, EntryPosition rollbackPosition) {
    try {
        List<MetaHistoryDO> metaHistoryDOList = metaHistoryDAO.findByTimestamp(destination,
            position.getTimestamp(),
            rollbackPosition.getTimestamp());
        if (metaHistoryDOList == null) {
            return true;
        }

        for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
            String binlogFile = metaHistoryDO.getBinlogFile();
            Long binlogOffest = metaHistoryDO.getBinlogOffest();
            String binlogMasterId = metaHistoryDO.getBinlogMasterId();
            Long binlogTimestamp = metaHistoryDO.getBinlogTimestamp();
            String useSchema = metaHistoryDO.getUseSchema();
            String sqlData = metaHistoryDO.getSqlText();
            EntryPosition snapshotPosition = new EntryPosition(binlogFile,
                binlogOffest == null ? 0L : binlogOffest,
                binlogTimestamp == null ? 0L : binlogTimestamp,
                Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));

            // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
            if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
                continue;
            } else if (rollbackPosition.getServerId() == snapshotPosition.getServerId()
                       && snapshotPosition.compareTo(rollbackPosition) > 0) {
                continue;
            }

            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData, null)) {
                return false;
            }

        }

        return metaHistoryDOList.size() > 0;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed", e);
    }
}

Example 8

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

4 votes

private boolean applyHistoryOnMemory(EntryPosition position, EntryPosition rollbackPosition) {
    try {
        List<MetaHistoryDO> metaHistoryDOList = metaHistoryDAO.findByTimestamp(destination,
            position.getTimestamp(),
            rollbackPosition.getTimestamp());
        if (metaHistoryDOList == null) {
            return true;
        }

        for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
            String binlogFile = metaHistoryDO.getBinlogFile();
            Long binlogOffest = metaHistoryDO.getBinlogOffest();
            String binlogMasterId = metaHistoryDO.getBinlogMasterId();
            Long binlogTimestamp = metaHistoryDO.getBinlogTimestamp();
            String useSchema = metaHistoryDO.getUseSchema();
            String sqlData = metaHistoryDO.getSqlText();
            EntryPosition snapshotPosition = new EntryPosition(binlogFile,
                binlogOffest == null ? 0L : binlogOffest,
                binlogTimestamp == null ? 0L : binlogTimestamp,
                Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));

            // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
            if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
                continue;
            } else if (rollbackPosition.getServerId() == snapshotPosition.getServerId()
                       && snapshotPosition.compareTo(rollbackPosition) > 0) {
                continue;
            }

            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData, null)) {
                return false;
            }

        }

        return metaHistoryDOList.size() > 0;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed", e);
    }
}

Example 9

Source File: MysqlEventParser.java From canal with Apache License 2.0

4 votes

protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = null;
        if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
            entryPosition = masterPosition;
        } else if (standbyInfo != null
                   && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
            entryPosition = standbyPosition;
        }

        if (entryPosition == null) {
            entryPosition = findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
        }

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { "", "", entryPosition.getTimestamp() });
                return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
            } else {
                logger.warn("prepare to find start position just show master status");
                return findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
            }
        } else {
            if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                // 如果指定binlogName + offest，直接返回
                entryPosition = findPositionWithMasterIdAndTimestamp(mysqlConnection, entryPosition);
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(),
                            entryPosition.getTimestamp() });
                return entryPosition;
            } else {
                EntryPosition specificLogFilePosition = null;
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    // 如果指定binlogName +
                    // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
                    EntryPosition endPosition = findEndPosition(mysqlConnection);
                    if (endPosition != null) {
                        logger.warn("prepare to find start position {}:{}:{}",
                            new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
                        specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
                            entryPosition.getTimestamp(),
                            endPosition,
                            entryPosition.getJournalName(),
                            true);
                    }
                }

                if (specificLogFilePosition == null) {
                    // position不存在，从文件头开始
                    entryPosition.setPosition(BINLOG_START_OFFEST);
                    return entryPosition;
                } else {
                    return specificLogFilePosition;
                }
            }
        }
    } else {
        if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
            if (dumpErrorCountThreshold >= 0 && dumpErrorCount > dumpErrorCountThreshold) {
                // binlog定位位点失败,可能有两个原因:
                // 1. binlog位点被删除
                // 2.vip模式的mysql,发生了主备切换,判断一下serverId是否变化,针对这种模式可以发起一次基于时间戳查找合适的binlog位点
                boolean case2 = (standbyInfo == null || standbyInfo.getAddress() == null)
                                && logPosition.getPostion().getServerId() != null
                                && !logPosition.getPostion().getServerId().equals(findServerId(mysqlConnection));
                if (case2) {
                    EntryPosition findPosition = fallbackFindByStartTimestamp(logPosition, mysqlConnection);
                    dumpErrorCount = 0;
                    return findPosition;
                }

                Long timestamp = logPosition.getPostion().getTimestamp();
                if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
                    // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
                    return null;
                }
            } else if (StringUtils.isBlank(logPosition.getPostion().getJournalName())
                    && logPosition.getPostion().getPosition() <= 0
                    && logPosition.getPostion().getTimestamp() > 0) {
                return fallbackFindByStartTimestamp(logPosition,mysqlConnection);
            }
            // 其余情况
            logger.warn("prepare to find start position just last position\n {}",
                JsonUtils.marshalToString(logPosition));
            return logPosition.getPostion();
        } else {
            // 针对切换的情况，考虑回退时间
            long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
            logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "",
                    logPosition.getPostion().getTimestamp() });
            return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
        }
    }
}