com.alibaba.otter.canal.protocol.position.EntryPosition Java Examples

The following examples show how to use com.alibaba.otter.canal.protocol.position.EntryPosition. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #2

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) {
        // 使用一个未来极大的时间，基于位点进行定位
        long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年
        EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
            startTimestamp,
            fixedPosition,
            fixedPosition.getJournalName(),
            true);
        if (entryPosition == null) {
            throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position"
                                          + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition());
        }
        return entryPosition;
    } else {
        return fixedPosition;
    }
}

Example #3

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #4

Source File: CanalEventUtils.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(event.getServerId());
    // add gtid
    position.setGtid(event.getGtid());

    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}

Example #5

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected EntryPosition findPositionWithMasterIdAndTimestamp(MysqlConnection connection, EntryPosition fixedPosition) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    if (tableMetaTSDB != null && (fixedPosition.getTimestamp() == null || fixedPosition.getTimestamp() <= 0)) {
        // 使用一个未来极大的时间，基于位点进行定位
        long startTimestamp = System.currentTimeMillis() + 102L * 365 * 24 * 3600 * 1000; // 当前时间的未来102年
        EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
            startTimestamp,
            fixedPosition,
            fixedPosition.getJournalName(),
            true);
        if (entryPosition == null) {
            throw new CanalParseException("[fixed timestamp] can't found begin/commit position before with fixed position"
                                          + fixedPosition.getJournalName() + ":" + fixedPosition.getPosition());
        }
        return entryPosition;
    } else {
        return fixedPosition;
    }
}

Example #6

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

/**
 * 查询当前的binlog位置
 */
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show master status");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        if (isGTIDMode() && fields.size() > 4) {
            endPosition.setGtid(fields.get(4));
        }
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show master status' has an error!", e);
    }
}

Example #7

Source File: MysqlEventParser.java From canal with Apache License 2.0

6 votes

/**
 * 查询当前的binlog位置
 */
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show master status");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        if (isGTIDMode() && fields.size() > 4) {
            endPosition.setGtid(fields.get(4));
        }
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show master status' has an error!", e);
    }
}

Example #8

Source File: AbstractEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    // set gtid
    position.setGtid(entry.getHeader().getGtid());

    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}

Example #9

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

6 votes

@Override
public boolean rollback(EntryPosition position) {
    // 每次rollback需要重新构建一次memory data
    this.memoryTableMeta = new MemoryTableMeta();
    boolean flag = false;
    EntryPosition snapshotPosition = buildMemFromSnapshot(position);
    if (snapshotPosition != null) {
        applyHistoryOnMemory(snapshotPosition, position);
        flag = true;
    }

    if (!flag) {
        // 如果没有任何数据，则为初始化状态，全量dump一份关注的表
        if (dumpTableMeta(connection, filter)) {
            // 记录一下snapshot结果,方便快速恢复
            flag = applySnapshotToDB(INIT_POSITION, true);
        }
    }

    return flag;
}

Example #10

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public boolean rollback(EntryPosition position) {
    // 每次rollback需要重新构建一次memory data
    this.memoryTableMeta = new MemoryTableMeta();
    boolean flag = false;
    EntryPosition snapshotPosition = buildMemFromSnapshot(position);
    if (snapshotPosition != null) {
        applyHistoryOnMemory(snapshotPosition, position);
        flag = true;
    }

    if (!flag) {
        // 如果没有任何数据，则为初始化状态，全量dump一份关注的表
        if (dumpTableMeta(connection, filter)) {
            // 记录一下snapshot结果,方便快速恢复
            flag = applySnapshotToDB(INIT_POSITION, true);
        }
    }

    return flag;
}

Example #11

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
    // 首先记录到内存结构
    lock.writeLock().lock();
    try {
        if (memoryTableMeta.apply(position, schema, ddl, extra)) {
            this.lastPosition = position;
            this.hasNewDdl = true;
            // 同步每次变更给远程做历史记录
            return applyHistoryToDB(position, schema, ddl, extra);
        } else {
            throw new RuntimeException("apply to memory is failed");
        }
    } finally {
        lock.writeLock().unlock();
    }
}

Example #12

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

6 votes

@Override
public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
    // 首先记录到内存结构
    lock.writeLock().lock();
    try {
        if (memoryTableMeta.apply(position, schema, ddl, extra)) {
            this.lastPosition = position;
            this.hasNewDdl = true;
            // 同步每次变更给远程做历史记录
            return applyHistoryToDB(position, schema, ddl, extra);
        } else {
            throw new RuntimeException("apply to memory is failed");
        }
    } finally {
        lock.writeLock().unlock();
    }
}

Example #13

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

/**
 * 回滚到指定位点
 * 
 * @param position
 * @return
 */
protected boolean processTableMeta(EntryPosition position) {
    if (tableMetaTSDB != null) {
        if (position.getTimestamp() == null || position.getTimestamp() <= 0) {
            throw new CanalParseException("use gtid and TableMeta TSDB should be config timestamp > 0");
        }

        return tableMetaTSDB.rollback(position);
    }

    return true;
}

Example #14

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

5 votes

/**
 * 更新一下本地的表结构内存
 *
 * @param position
 * @param schema
 * @param ddl
 * @return
 */
public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
    if (tableMetaTSDB != null) {
        return tableMetaTSDB.apply(position, schema, ddl, extra);
    } else {
        // ignore
        return true;
    }
}

Example #15

Source File: MysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
    if (isGTIDMode()) {
        // GTID模式下，CanalLogPositionManager里取最后的gtid，没有则取instanc配置中的
        LogPosition logPosition = getLogPositionManager().getLatestIndexBy(destination);
        if (logPosition != null) {
            return logPosition.getPostion();
        }

        if (masterPosition != null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
            return masterPosition;
        }
    }

    EntryPosition startPosition = findStartPositionInternal(connection);
    if (needTransactionPosition.get()) {
        logger.warn("prepare to find last position : {}", startPosition.toString());
        Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
        if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
            logger.warn("find new start Transaction Position , old : {} , new : {}",
                startPosition.getPosition(),
                preTransactionStartPosition);
            startPosition.setPosition(preTransactionStartPosition);
        }
        needTransactionPosition.compareAndSet(true, false);
    }
    return startPosition;
}

Example #16

Source File: TableMetaCache.java From canal with Apache License 2.0

5 votes

/**
 * 更新一下本地的表结构内存
 *
 * @param position
 * @param schema
 * @param ddl
 * @return
 */
public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
    if (tableMetaTSDB != null) {
        return tableMetaTSDB.apply(position, schema, ddl, extra);
    } else {
        // ignore
        return true;
    }
}

Example #17

Source File: MemoryTableMeta.java From canal with Apache License 2.0

5 votes

public boolean apply(EntryPosition position, String schema, String ddl, String extra) {
    tableMetas.clear();
    synchronized (this) {
        if (StringUtils.isNotEmpty(schema)) {
            repository.setDefaultSchema(schema);
        }

        try {
            // druid暂时flush privileges语法解析有问题
            if (!StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "flush")
                && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "grant")
                && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "revoke")
                && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create user")
                && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "alter user")
                && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "drop user")
                && !StringUtils.startsWithIgnoreCase(StringUtils.trim(ddl), "create database")) {
                repository.console(ddl);
            }
        } catch (Throwable e) {
            logger.warn("parse faield : " + ddl, e);
        }
    }

    // TableMeta meta = find("tddl5_00", "ab");
    // if (meta != null) {
    // repository.setDefaultSchema("tddl5_00");
    // System.out.println(repository.console("show create table tddl5_00.ab"));
    // System.out.println(repository.console("show columns from tddl5_00.ab"));
    // }
    return true;
}

Example #18

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

private EntryPosition buildMemFromSnapshot(EntryPosition position) {
    try {
        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp());
        if (snapshotDO == null) {
            return null;
        }
        String binlogFile = snapshotDO.getBinlogFile();
        Long binlogOffest = snapshotDO.getBinlogOffest();
        String binlogMasterId = snapshotDO.getBinlogMasterId();
        Long binlogTimestamp = snapshotDO.getBinlogTimestamp();

        EntryPosition snapshotPosition = new EntryPosition(binlogFile,
            binlogOffest == null ? 0l : binlogOffest,
            binlogTimestamp == null ? 0l : binlogTimestamp,
            Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
        // data存储为Map<String,String>，每个分库一套建表
        String sqlData = snapshotDO.getData();
        JSONObject jsonObj = JSON.parseObject(sqlData);
        for (Map.Entry entry : jsonObj.entrySet()) {
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition,
                ObjectUtils.toString(entry.getKey()),
                ObjectUtils.toString(entry.getValue()),
                null)) {
                return null;
            }
        }

        return snapshotPosition;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
    }
}

Example #19

Source File: LocalBinlogEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}

Example #20

Source File: MetaLogPositionManagerTest.java From canal-1.1.3 with Apache License 2.0

5 votes

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}

Example #21

Source File: CanalEventUtils.java From canal-1.1.3 with Apache License 2.0

5 votes

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event, boolean included) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    position.setIncluded(included);

    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}

Example #22

Source File: CanalEventUtils.java From canal-1.1.3 with Apache License 2.0

5 votes

/**
 * 判断当前的entry和position是否相同
 */
public static boolean checkPosition(Event event, LogPosition logPosition) {
    EntryPosition position = logPosition.getPostion();
    boolean result = position.getTimestamp().equals(event.getExecuteTime());

    boolean exactely = (StringUtils.isBlank(position.getJournalName()) && position.getPosition() == null);
    if (!exactely) {// 精确匹配
        result &= position.getPosition().equals(event.getPosition());
        if (result) {// short path
            result &= StringUtils.equals(event.getJournalName(), position.getJournalName());
        }
    }

    return result;
}

Example #23

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

5 votes

private EntryPosition buildMemFromSnapshot(EntryPosition position) {
    try {
        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp());
        if (snapshotDO == null) {
            return null;
        }
        String binlogFile = snapshotDO.getBinlogFile();
        Long binlogOffest = snapshotDO.getBinlogOffest();
        String binlogMasterId = snapshotDO.getBinlogMasterId();
        Long binlogTimestamp = snapshotDO.getBinlogTimestamp();

        EntryPosition snapshotPosition = new EntryPosition(binlogFile,
            binlogOffest == null ? 0l : binlogOffest,
            binlogTimestamp == null ? 0l : binlogTimestamp,
            Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
        // data存储为Map<String,String>，每个分库一套建表
        String sqlData = snapshotDO.getData();
        JSONObject jsonObj = JSON.parseObject(sqlData);
        for (Map.Entry entry : jsonObj.entrySet()) {
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition,
                ObjectUtils.toString(entry.getKey()),
                ObjectUtils.toString(entry.getValue()),
                null)) {
                return null;
            }
        }

        return snapshotPosition;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
    }
}

Example #24

Source File: MysqlTaskPositionManager.java From DataLink with Apache License 2.0

5 votes

public LogPosition translateToLogPosition(MysqlReaderPosition cp) {
    LogIdentity logIdentity = new LogIdentity(cp.getSourceAddress(), cp.getSlaveId());

    EntryPosition entryPosition = new EntryPosition(cp.getJournalName(), cp.getPosition(), cp.getTimestamp(), cp.getServerId());
    entryPosition.setIncluded(cp.isIncluded());
    entryPosition.setGtid(cp.getGtid());

    LogPosition logPosition = new LogPosition();
    logPosition.setIdentity(logIdentity);
    logPosition.setPostion(entryPosition);

    return logPosition;
}

Example #25

Source File: CanalEventUtils.java From canal with Apache License 2.0

5 votes

/**
 * 判断当前的entry和position是否相同
 */
public static boolean checkPosition(Event event, LogPosition logPosition) {
    EntryPosition position = logPosition.getPostion();
    boolean result = position.getTimestamp().equals(event.getExecuteTime());

    boolean exactely = (StringUtils.isBlank(position.getJournalName()) && position.getPosition() == null);
    if (!exactely) {// 精确匹配
        result &= position.getPosition().equals(event.getPosition());
        if (result) {// short path
            result &= StringUtils.equals(event.getJournalName(), position.getJournalName());
        }
    }

    return result;
}

Example #26

Source File: AbstractEventParser.java From DBus with Apache License 2.0

5 votes

protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}

Example #27

Source File: Binlog.java From jlogstash-input-plugin with Apache License 2.0

5 votes

@Override
public void prepare() {
    logger.info("binlog prepare started..");

    parseCategories();

    controller = new MysqlEventParser();
    controller.setConnectionCharset(Charset.forName("UTF-8"));
    controller.setSlaveId(slaveId);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress(host, port), username, password));
    controller.setEnableTsdb(true);
    controller.setDestination("example");
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(2);
    controller.setIsGTIDMode(false);
    controller.setEventSink(new BinlogEventSink(this));

    controller.setLogPositionManager(new BinlogPositionManager(this));

    EntryPosition startPosition = findStartPosition();
    if (startPosition != null) {
       controller.setMasterPosition(startPosition);
    }


    if (filter != null) {
        controller.setEventFilter(new AviaterRegexFilter(filter));
    }

    logger.info("binlog prepare ended..");
}

Example #28

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

5 votes

private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) {
    Map<String, String> content = new HashMap<String, String>();
    content.put("destination", destination);
    content.put("binlogFile", position.getJournalName());
    content.put("binlogOffest", String.valueOf(position.getPosition()));
    content.put("binlogMasterId", String.valueOf(position.getServerId()));
    content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
    content.put("useSchema", schema);
    if (content.isEmpty()) {
        throw new RuntimeException("apply failed caused by content is empty in applyHistoryToDB");
    }
    // 待补充
    List<DdlResult> ddlResults = DruidDdlParser.parse(ddl, schema);
    if (ddlResults.size() > 0) {
        DdlResult ddlResult = ddlResults.get(0);
        content.put("sqlSchema", ddlResult.getSchemaName());
        content.put("sqlTable", ddlResult.getTableName());
        content.put("sqlType", ddlResult.getType().name());
        content.put("sqlText", ddl);
        content.put("extra", extra);
    }

    MetaHistoryDO metaDO = new MetaHistoryDO();
    try {
        BeanUtils.populate(metaDO, content);
        // 会建立唯一约束,解决:
        // 1. 重复的binlog file+offest
        // 2. 重复的masterId+timestamp
        metaHistoryDAO.insert(metaDO);
    } catch (Throwable e) {
        if (isUkDuplicateException(e)) {
            // 忽略掉重复的位点
            logger.warn("dup apply for sql : " + ddl);
        } else {
            throw new CanalParseException("apply history to db failed caused by : " + e.getMessage(), e);
        }

    }
    return true;
}

Example #29

Source File: AbstractMetaManagerTest.java From canal with Apache License 2.0

5 votes

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}

Example #30

Source File: LocalBinlogEventParser.java From canal with Apache License 2.0

5 votes

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}