Java Code Examples for com.alibaba.otter.canal.protocol.position.LogPosition

The following examples show how to use com.alibaba.otter.canal.protocol.position.LogPosition. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 
Example 2
Source Project: canal-1.1.3   Source File: MetaLogPositionManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 
Example 3
Source Project: canal-1.1.3   Source File: MixedLogPositionManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);

    MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1000);

    MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 
Example 4
Source Project: canal   Source File: AbstractCanalStoreScavenge.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 
Example 5
@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();

    FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager(dataDir,
        1000,
        memoryLogPositionManager);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1500);

    FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager(dataDir,
        1000,
        memoryLogPositionManager);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 
Example 6
Source Project: canal-1.1.3   Source File: MemoryEventStoreWithBuffer.java    License: Apache License 2.0 6 votes vote down vote up
public Events<Event> get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkUnGetSlotAt((LogPosition) start, batchSize))
                notEmpty.await();
        } catch (InterruptedException ie) {
            notEmpty.signal(); // propagate to non-interrupted thread
            throw ie;
        }

        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}
 
Example 7
Source Project: canal-1.1.3   Source File: CanalEventUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 找出一个最小的position位置,相等的情况返回position1
 */
public static LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() > position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库,根据时间进行比较
        if (position1.getPostion().getTimestamp() > position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 
Example 8
Source Project: canal   Source File: MixedLogPositionManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);

    MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1000);

    MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 
Example 9
Source Project: canal-1.1.3   Source File: AbstractCanalStoreScavenge.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 
Example 10
Source Project: canal-1.1.3   Source File: AbstractCanalStoreScavenge.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 找出一个最小的position位置
 */
private LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库,根据时间进行比较
        if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 
Example 11
Source Project: DataLink   Source File: GatewayMetaManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 需要加同步锁,保证在获取最小Position的时候,metaManager是不可变的
 *
 * @throws CanalMetaManagerException
 */
@Override
public synchronized Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    // 入参clientIdentity没有什么作用,因为该方法需要返回所有"Sub Meta Manager"中最小的Position
    List<ClientIdentity> clientIdentities = listAllSubscribeInfo("");
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity item : clientIdentities) {
            LogPosition position = (LogPosition) attachedMetaManagers.get(item.getDestination()).getCursor(item);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 
Example 12
Source Project: DBus   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 
Example 13
Source Project: canal   Source File: MetaLogPositionManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 
Example 14
Source Project: canal   Source File: AbstractCanalStoreScavenge.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 找出一个最小的position位置
 */
private LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库,根据时间进行比较
        if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 
Example 15
Source Project: canal   Source File: AbstractLogPositionManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
public LogPosition doTest(CanalLogPositionManager logPositionManager) {
    LogPosition getPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertNull(getPosition);

    LogPosition postion1 = buildPosition(1);
    logPositionManager.persistLogPosition(destination, postion1);
    LogPosition getPosition1 = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(postion1, getPosition1);

    LogPosition postion2 = buildPosition(2);
    logPositionManager.persistLogPosition(destination, postion2);
    LogPosition getPosition2 = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(postion2, getPosition2);
    return postion2;
}
 
Example 16
Source Project: canal   Source File: MixedLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
    memoryLogPositionManager.persistLogPosition(destination, logPosition);
    executor.submit(new Runnable() {

        public void run() {
            try {
                zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
            } catch (Exception e) {
                logger.error("ERROR # persist to zookeeper has an error", e);
            }
        }
    });
}
 
Example 17
Source Project: canal-1.1.3   Source File: AbstractMetaManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 
Example 18
Source Project: canal   Source File: PeriodMixedLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
    if (logPosition == nullPosition) {
        return null;
    } else {
        return logPosition;
    }
}
 
Example 19
Source Project: canal-1.1.3   Source File: LocalBinlogEventParser.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录,判断一下当前记录是否发生过主备切换
    // // a. 无机器切换,直接返回
    // // b. 存在机器切换,按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录,则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName,尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest,直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}
 
Example 20
Source Project: canal-1.1.3   Source File: MysqlEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
    if (isGTIDMode()) {
        // GTID模式下,CanalLogPositionManager里取最后的gtid,没有则取instanc配置中的
        LogPosition logPosition = getLogPositionManager().getLatestIndexBy(destination);
        if (logPosition != null) {
            return logPosition.getPostion();
        }

        if (masterPosition != null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
            return masterPosition;
        }
    }

    EntryPosition startPosition = findStartPositionInternal(connection);
    if (needTransactionPosition.get()) {
        logger.warn("prepare to find last position : {}", startPosition.toString());
        Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
        if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
            logger.warn("find new start Transaction Position , old : {} , new : {}",
                startPosition.getPosition(),
                preTransactionStartPosition);
            startPosition.setPosition(preTransactionStartPosition);
        }
        needTransactionPosition.compareAndSet(true, false);
    }
    return startPosition;
}
 
Example 21
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
 
Example 22
Source Project: canal   Source File: MixedLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
    if (logPosition != null) {
        return logPosition;
    }
    logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
    // 这里保持和重构前的逻辑一致,重新添加到Memory中
    if (logPosition != null) {
        memoryLogPositionManager.persistLogPosition(destination, logPosition);
    }
    return logPosition;
}
 
Example 23
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, Long startPosition) {
    if (lastPosition != null) {
        logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
            lastPosition.getPostion()),
            e);
    } else {
        logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s,%s]",
            startBinlogFile,
            startPosition), e);
    }
}
 
Example 24
Source Project: canal-1.1.3   Source File: ZooKeeperLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
 
Example 25
Source Project: canal-1.1.3   Source File: ZooKeeperLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}
 
Example 26
Source Project: canal   Source File: MemoryEventStoreWithBuffer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 检查是否存在需要get的数据,并且数量>=batchSize
 */
private boolean checkUnGetSlotAt(LogPosition startPosition, int batchSize) {
    if (batchMode.isItemSize()) {
        long current = getSequence.get();
        long maxAbleSequence = putSequence.get();
        long next = current;
        if (startPosition == null || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后,需要包含一下start位置,防止丢失第一条记录
            next = next + 1;// 少一条数据
        }

        if (current < maxAbleSequence && next + batchSize - 1 <= maxAbleSequence) {
            return true;
        } else {
            return false;
        }
    } else {
        // 处理内存大小判断
        long currentSize = getMemSize.get();
        long maxAbleSize = putMemSize.get();

        if (maxAbleSize - currentSize >= batchSize * bufferMemUnit) {
            return true;
        } else {
            return false;
        }
    }
}
 
Example 27
Source Project: canal-1.1.3   Source File: FailbackLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    try {
        primary.persistLogPosition(destination, logPosition);
    } catch (CanalParseException e) {
        logger.warn("persistLogPosition use primary log position manager exception. destination: {}, logPosition: {}",
            destination,
            logPosition,
            e);
        secondary.persistLogPosition(destination, logPosition);
    }
}
 
Example 28
Source Project: canal-1.1.3   Source File: MixedLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
    if (logPosition != null) {
        return logPosition;
    }
    logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
    // 这里保持和重构前的逻辑一致,重新添加到Memory中
    if (logPosition != null) {
        memoryLogPositionManager.persistLogPosition(destination, logPosition);
    }
    return logPosition;
}
 
Example 29
Source Project: canal-1.1.3   Source File: MixedLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void persistLogPosition(final String destination, final LogPosition logPosition) throws CanalParseException {
    memoryLogPositionManager.persistLogPosition(destination, logPosition);
    executor.submit(new Runnable() {

        public void run() {
            try {
                zooKeeperLogPositionManager.persistLogPosition(destination, logPosition);
            } catch (Exception e) {
                logger.error("ERROR # persist to zookeeper has an error", e);
            }
        }
    });
}
 
Example 30
Source Project: canal-1.1.3   Source File: PeriodMixedLogPositionManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
    if (logPosition == nullPosition) {
        return null;
    } else {
        return logPosition;
    }
}