com.alibaba.otter.canal.protocol.position.LogPosition Java Examples

@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);

    MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1000);

    MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 

/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 

/**
 * 找出一个最小的position位置
 */
private LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库，根据时间进行比较
        if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 

@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);

    MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1000);

    MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 

/**
 * 找出一个最小的position位置，相等的情况返回position1
 */
public static LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() > position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库，根据时间进行比较
        if (position1.getPostion().getTimestamp() > position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 

/**
 * 需要加同步锁，保证在获取最小Position的时候，metaManager是不可变的
 *
 * @throws CanalMetaManagerException
 */
@Override
public synchronized Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    // 入参clientIdentity没有什么作用，因为该方法需要返回所有"Sub Meta Manager"中最小的Position
    List<ClientIdentity> clientIdentities = listAllSubscribeInfo("");
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity item : clientIdentities) {
            LogPosition position = (LogPosition) attachedMetaManagers.get(item.getDestination()).getCursor(item);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 

@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 

public Events<Event> get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkUnGetSlotAt((LogPosition) start, batchSize))
                notEmpty.await();
        } catch (InterruptedException ie) {
            notEmpty.signal(); // propagate to non-interrupted thread
            throw ie;
        }

        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}
 

/**
 * 找出一个最小的position位置
 */
private LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库，根据时间进行比较
        if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 

@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();

    FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager(dataDir,
        1000,
        memoryLogPositionManager);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1500);

    FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager(dataDir,
        1000,
        memoryLogPositionManager);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 

/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 

@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 

private void registerInternal() {
    Map<String, Long> timeStamps = new HashMap<>();
    Map<String, RegisterItem> validInstances = new HashMap<>();
    Map<String, RegisterItem> copiedInstances = new HashMap<>(this.endpointInstances);
    copiedInstances.entrySet().forEach(e -> {
        LogPosition position = (LogPosition) e.getValue().instance.getMetaManager()
                .getCursor(new ClientIdentity(e.getKey(), (short) 1001, ""));
        timeStamps.put(e.getKey(), position == null ? System.currentTimeMillis() : position.getPostion().getTimestamp());
    });
    double avgTime = timeStamps.values().stream().mapToLong(p -> p).summaryStatistics().getAverage();
    copiedInstances.entrySet().forEach(e -> {
        //位点时间比平均时间小于两个小时以上时，不参与多路复用
        double diff = avgTime - timeStamps.get(e.getKey());
        if (diff > LESS_THAN_AVG_LOG_TIME_THRESHOLD) {
            kickout(e.getKey(), false,
                    String.format("exceeded threshold than the average log time, this time is [%s] ms, avg time is [%s] ms ," +
                                    " diff time is [%s] ms, threshold is [%s].",
                            timeStamps.get(e.getKey()), avgTime, diff, LESS_THAN_AVG_LOG_TIME_THRESHOLD));
        } else {
            validInstances.put(e.getKey(), e.getValue());
        }
    });


    validInstances.entrySet().forEach(e -> {
        ((GatewayEventSink) workingInstance.getEventSink())
                .registerEventSink(e.getKey(), e.getValue().instance.getEventSink());
        ((GatewayEventStore) workingInstance.getEventStore())
                .registerEventStore(e.getKey(), e.getValue().instance.getEventStore());
        ((GatewayMetaManager) workingInstance.getMetaManager())
                .registerMetaManager(e.getKey(), e.getValue().instance.getMetaManager());
    });
}
 

public LogPosition doTest(CanalLogPositionManager logPositionManager) {
    LogPosition getPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertNull(getPosition);

    LogPosition postion1 = buildPosition(1);
    logPositionManager.persistLogPosition(destination, postion1);
    LogPosition getPosition1 = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(postion1, getPosition1);

    LogPosition postion2 = buildPosition(2);
    logPositionManager.persistLogPosition(destination, postion2);
    LogPosition getPosition2 = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(postion2, getPosition2);
    return postion2;
}
 

private MysqlReaderPosition buildPosition(Position position) {
    MysqlReaderPosition mysqlReaderPosition = taskPositionManager.translateToMysqlReaderPosition((LogPosition) position);
    CanalReaderEffectSyncPosition effectSyncPosition = canalReader.getEffectSyncPosition();
    TaskShadowInfo taskShadow = canalReader.getTaskShadow();
    if (effectSyncPosition != null) {
        mysqlReaderPosition.setLatestEffectSyncLogFileName(effectSyncPosition.getLatestEffectSyncLogFileName());
        mysqlReaderPosition.setLatestEffectSyncLogFileOffset(effectSyncPosition.getLatestEffectSyncLogFileOffset());
    }
    if (taskShadow != null) {
        mysqlReaderPosition.setShadowId(taskShadow.getId());
    }
    return mysqlReaderPosition;
}
 

public LogPosition translateToLogPosition(MysqlReaderPosition cp) {
    LogIdentity logIdentity = new LogIdentity(cp.getSourceAddress(), cp.getSlaveId());

    EntryPosition entryPosition = new EntryPosition(cp.getJournalName(), cp.getPosition(), cp.getTimestamp(), cp.getServerId());
    entryPosition.setIncluded(cp.isIncluded());
    entryPosition.setGtid(cp.getGtid());

    LogPosition logPosition = new LogPosition();
    logPosition.setIdentity(logIdentity);
    logPosition.setPostion(entryPosition);

    return logPosition;
}
 

protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, long startPosition) {
    if (lastPosition != null) {
        logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
            lastPosition.getPostion()),
            e);
    } else {
        logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s,%s]",
            startBinlogFile,
            startPosition), e);
    }
}
 

@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录，判断一下当前记录是否发生过主备切换
    // // a. 无机器切换，直接返回
    // // b. 存在机器切换，按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录，则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest，直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}
 

private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
 

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());

    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }
    synchronized (canalInstance) {
        // 清除batch信息
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
            batchId);
        if (positionRanges == null) { // 说明是重复的ack/rollback
            throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
                clientIdentity.getClientId(),
                batchId));
        }

        // lastRollbackPostions.put(clientIdentity,
        // positionRanges.getEnd());// 记录一下最后rollback的位置
        // TODO 后续rollback到指定的batchId位置
        canalInstance.getEventStore().rollback();// rollback
                                                 // eventStore中的状态信息
        logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
            clientIdentity.getClientId(),
            batchId,
            positionRanges);
    }
}
 

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 

protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
 

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event, boolean included) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    position.setIncluded(included);

    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}
 

@Test
public void testAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();

    MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
    logPositionManager.start();
    // 构建meta信息
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);

    PositionRange range1 = buildRange(1);
    metaManager.updateCursor(client1, range1.getEnd());

    PositionRange range2 = buildRange(2);
    metaManager.updateCursor(client1, range2.getEnd());

    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);

    PositionRange range3 = buildRange(3);
    metaManager.updateCursor(client2, range3.getEnd());

    PositionRange range4 = buildRange(4);
    metaManager.updateCursor(client2, range4.getEnd());

    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(range2.getEnd(), logPosition);

    metaManager.stop();
    logPositionManager.stop();
}
 

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 

/**
 * 检查是否存在需要get的数据,并且数量>=batchSize
 */
private boolean checkUnGetSlotAt(LogPosition startPosition, int batchSize) {
    if (batchMode.isItemSize()) {
        long current = getSequence.get();
        long maxAbleSequence = putSequence.get();
        long next = current;
        if (startPosition == null || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后，需要包含一下start位置，防止丢失第一条记录
            next = next + 1;// 少一条数据
        }

        if (current < maxAbleSequence && next + batchSize - 1 <= maxAbleSequence) {
            return true;
        } else {
            return false;
        }
    } else {
        // 处理内存大小判断
        long currentSize = getMemSize.get();
        long maxAbleSize = putMemSize.get();

        if (maxAbleSize - currentSize >= batchSize * bufferMemUnit) {
            return true;
        } else {
            return false;
        }
    }
}
 

@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
 

private void flushDataToFile(String destination, File dataFile) {
    LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
    if (position != null && position != nullPosition) {
        String json = JsonUtils.marshalToString(position);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}