Java Code Examples for com.alibaba.otter.canal.protocol.position.EntryPosition#getPosition()

The following examples show how to use com.alibaba.otter.canal.protocol.position.EntryPosition#getPosition() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) {
        // 使用一个未来极大的时间，基于位点进行定位
        long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年
        EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
            startTimestamp,
            fixedPosition,
            fixedPosition.getJournalName(),
            true);
        if (entryPosition == null) {
            throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position"
                                          + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition());
        }
        return entryPosition;
    } else {
        return fixedPosition;
    }
}

Example 2

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) {
        // 使用一个未来极大的时间，基于位点进行定位
        long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年
        EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
            startTimestamp,
            fixedPosition,
            fixedPosition.getJournalName(),
            true);
        if (entryPosition == null) {
            throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position"
                                          + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition());
        }
        return entryPosition;
    } else {
        return fixedPosition;
    }
}

Example 3

Source File: LocalBinlogEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}

Example 4

Source File: CanalEventUtils.java From canal-1.1.3 with Apache License 2.0

5 votes

/**
 * 判断当前的entry和position是否相同
 */
public static boolean checkPosition(Event event, LogPosition logPosition) {
    EntryPosition position = logPosition.getPostion();
    boolean result = position.getTimestamp().equals(event.getExecuteTime());

    boolean exactely = (StringUtils.isBlank(position.getJournalName()) && position.getPosition() == null);
    if (!exactely) {// 精确匹配
        result &= position.getPosition().equals(event.getPosition());
        if (result) {// short path
            result &= StringUtils.equals(event.getJournalName(), position.getJournalName());
        }
    }

    return result;
}

Example 5

Source File: LocalBinlogEventParser.java From canal with Apache License 2.0

5 votes

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}

Example 6

Source File: CanalEventUtils.java From canal with Apache License 2.0

5 votes

/**
 * 判断当前的entry和position是否相同
 */
public static boolean checkPosition(Event event, LogPosition logPosition) {
    EntryPosition position = logPosition.getPostion();
    boolean result = position.getTimestamp().equals(event.getExecuteTime());

    boolean exactely = (StringUtils.isBlank(position.getJournalName()) && position.getPosition() == null);
    if (!exactely) {// 精确匹配
        result &= position.getPosition().equals(event.getPosition());
        if (result) {// short path
            result &= StringUtils.equals(event.getJournalName(), position.getJournalName());
        }
    }

    return result;
}

Example 7

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

4 votes

private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition)
                                                                                                             throws IOException {
    // 针对开始的第一条为非Begin记录，需要从该binlog扫描
    final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
    mysqlConnection.reconnect();
    mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction<LogEvent>() {

        private LogPosition lastPosition;

        public boolean sink(LogEvent event) {
            try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
                if (entry == null) {
                    return true;
                }

                // 直接查询第一条业务数据，确认是否为事务Begin
                // 记录一下transaction begin position
                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
                    && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
                    preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
                }

                if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
                    return false;// 退出
                }

                lastPosition = buildLastPosition(entry);
            } catch (Exception e) {
                processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                return false;
            }

            return running;
        }
    });

    // 判断一下找到的最接近position的事务头的位置
    if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
        logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
        throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
    }
    return preTransactionStartPosition.get();
}

Example 8

Source File: MysqlEventParser.java From canal with Apache License 2.0

4 votes

protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = null;
        if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
            entryPosition = masterPosition;
        } else if (standbyInfo != null
                   && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
            entryPosition = standbyPosition;
        }

        if (entryPosition == null) {
            entryPosition = findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
        }

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { "", "", entryPosition.getTimestamp() });
                return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
            } else {
                logger.warn("prepare to find start position just show master status");
                return findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
            }
        } else {
            if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                // 如果指定binlogName + offest，直接返回
                entryPosition = findPositionWithMasterIdAndTimestamp(mysqlConnection, entryPosition);
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(),
                            entryPosition.getTimestamp() });
                return entryPosition;
            } else {
                EntryPosition specificLogFilePosition = null;
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    // 如果指定binlogName +
                    // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
                    EntryPosition endPosition = findEndPosition(mysqlConnection);
                    if (endPosition != null) {
                        logger.warn("prepare to find start position {}:{}:{}",
                            new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
                        specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
                            entryPosition.getTimestamp(),
                            endPosition,
                            entryPosition.getJournalName(),
                            true);
                    }
                }

                if (specificLogFilePosition == null) {
                    // position不存在，从文件头开始
                    entryPosition.setPosition(BINLOG_START_OFFEST);
                    return entryPosition;
                } else {
                    return specificLogFilePosition;
                }
            }
        }
    } else {
        if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
            if (dumpErrorCountThreshold >= 0 && dumpErrorCount > dumpErrorCountThreshold) {
                // binlog定位位点失败,可能有两个原因:
                // 1. binlog位点被删除
                // 2.vip模式的mysql,发生了主备切换,判断一下serverId是否变化,针对这种模式可以发起一次基于时间戳查找合适的binlog位点
                boolean case2 = (standbyInfo == null || standbyInfo.getAddress() == null)
                                && logPosition.getPostion().getServerId() != null
                                && !logPosition.getPostion().getServerId().equals(findServerId(mysqlConnection));
                if (case2) {
                    EntryPosition findPosition = fallbackFindByStartTimestamp(logPosition, mysqlConnection);
                    dumpErrorCount = 0;
                    return findPosition;
                }

                Long timestamp = logPosition.getPostion().getTimestamp();
                if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
                    // 如果binlog位点不存在，并且属于timestamp不为空,可以返回null走到oss binlog处理
                    return null;
                }
            } else if (StringUtils.isBlank(logPosition.getPostion().getJournalName())
                    && logPosition.getPostion().getPosition() <= 0
                    && logPosition.getPostion().getTimestamp() > 0) {
                return fallbackFindByStartTimestamp(logPosition,mysqlConnection);
            }
            // 其余情况
            logger.warn("prepare to find start position just last position\n {}",
                JsonUtils.marshalToString(logPosition));
            return logPosition.getPostion();
        } else {
            // 针对切换的情况，考虑回退时间
            long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
            logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "",
                    logPosition.getPostion().getTimestamp() });
            return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
        }
    }
}

Example 9

Source File: MysqlEventParser.java From canal with Apache License 2.0

4 votes

private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition)
                                                                                                             throws IOException {
    // 针对开始的第一条为非Begin记录，需要从该binlog扫描
    final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
    mysqlConnection.reconnect();
    mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction<LogEvent>() {

        private LogPosition lastPosition;

        public boolean sink(LogEvent event) {
            try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
                if (entry == null) {
                    return true;
                }

                // 直接查询第一条业务数据，确认是否为事务Begin
                // 记录一下transaction begin position
                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN
                    && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
                    preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
                }

                if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
                    return false;// 退出
                }

                lastPosition = buildLastPosition(entry);
            } catch (Exception e) {
                processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                return false;
            }

            return running;
        }
    });

    // 判断一下找到的最接近position的事务头的位置
    if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
        logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
        throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
    }
    return preTransactionStartPosition.get();
}