com.alibaba.otter.canal.protocol.position.PositionRange Java Examples

@Test
public void testBatchAll() {
    PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

@Test
public void testBatchAll() {
    PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

public Position doCursorTest(CanalMetaManager metaManager) {
    metaManager.subscribe(clientIdentity);

    Position position1 = metaManager.getCursor(clientIdentity);
    Assert.assertNull(position1);

    PositionRange range = buildRange(1);

    metaManager.updateCursor(clientIdentity, range.getStart());
    Position position2 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getStart(), position2);

    metaManager.updateCursor(clientIdentity, range.getEnd());
    Position position3 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getEnd(), position3);

    return position3;
}
 

@Test
public void testBatchAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    doBatchTest(metaManager);

    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    MixedMetaManager metaManager2 = new MixedMetaManager();
    metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager2.start();

    Map<Long, PositionRange> ranges = metaManager2.listAllBatchs(clientIdentity);
    Assert.assertEquals(3, ranges.size());

    metaManager.clearAllBatchs(clientIdentity);
    ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
    metaManager2.stop();
}
 

public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    PositionRange result = getBatch(clientIdentity, minBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getFirstBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long maxBatchId = Collections.max(batchIds);
    PositionRange result = getBatch(clientIdentity, maxBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getLastestBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public Position doCursorTest(CanalMetaManager metaManager) {
    metaManager.subscribe(clientIdentity);

    Position position1 = metaManager.getCursor(clientIdentity);
    Assert.assertNull(position1);

    PositionRange range = buildRange(1);

    metaManager.updateCursor(clientIdentity, range.getStart());
    Position position2 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getStart(), position2);

    metaManager.updateCursor(clientIdentity, range.getEnd());
    Position position3 = metaManager.getCursor(clientIdentity);
    Assert.assertEquals(range.getEnd(), position3);

    return position3;
}
 

public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    PositionRange result = getBatch(clientIdentity, minBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getFirstBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long maxBatchId = Collections.max(batchIds);
    PositionRange result = getBatch(clientIdentity, maxBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getLastestBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public synchronized PositionRange getFirstPositionRange() {
    if (batches.size() == 0) {
        return null;
    } else {
        Long batchId = Collections.min(batches.keySet());
        return batches.get(batchId);
    }
}
 

public synchronized PositionRange removePositionRange(Long batchId) {
    if (batches.containsKey(batchId)) {
        Long minBatchId = Collections.min(batches.keySet());
        if (!minBatchId.equals(batchId)) {
            // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
            throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d",
                batchId,
                minBatchId));
        }
        return batches.remove(batchId);
    } else {
        return null;
    }
}
 

public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = zkClientx.getChildren(batchsPath);
    if (CollectionUtils.isEmpty(nodes)) {
        // 没有batch记录
        return null;
    }

    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    if (!minBatchId.equals(batchId)) {
        // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
        throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
    }

    if (!batchIds.contains(batchId)) {
        // 不存在对应的batchId
        return null;
    }
    PositionRange positionRange = getBatch(clientIdentity, batchId);
    if (positionRange != null) {
        String path = ZookeeperPathUtils
            .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
        zkClientx.delete(path);
    }

    return positionRange;
}
 

public synchronized PositionRange getLastestPositionRange() {
    if (batches.size() == 0) {
        return null;
    } else {
        Long batchId = Collections.max(batches.keySet());
        return batches.get(batchId);
    }
}
 

@Test
public void testBatchAll() {
    MemoryMetaManager metaManager = new MemoryMetaManager();
    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return Maps.newHashMap();
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }

    Collections.sort(batchIds); // 从小到大排序
    Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap();
    for (Long batchId : batchIds) {
        PositionRange result = getBatch(clientIdentity, batchId);
        if (result == null) {// 出现为null，说明zk节点有变化，重新获取
            return listAllBatchs(clientIdentity);
        } else {
            positionRanges.put(batchId, result);
        }
    }

    return positionRanges;
}
 

@Test
public void testAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();

    MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
    logPositionManager.start();
    // 构建meta信息
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);

    PositionRange range1 = buildRange(1);
    metaManager.updateCursor(client1, range1.getEnd());

    PositionRange range2 = buildRange(2);
    metaManager.updateCursor(client1, range2.getEnd());

    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);

    PositionRange range3 = buildRange(3);
    metaManager.updateCursor(client2, range3.getEnd());

    PositionRange range4 = buildRange(4);
    metaManager.updateCursor(client2, range4.getEnd());

    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(range2.getEnd(), logPosition);

    metaManager.stop();
    logPositionManager.stop();
}
 

public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange)
                                                                                            throws CanalMetaManagerException {
    final Long batchId = super.addBatch(clientIdentity, positionRange);
    // 异步刷新
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
        }
    });
    return batchId;
}
 

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 

@Test
public void testBatchAll() {
    MemoryMetaManager metaManager = new MemoryMetaManager();
    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

@Test
public void testBatchAll() {
    FileMixedMetaManager metaManager = new FileMixedMetaManager();
    metaManager.setDataDirByFile(dataDir);
    metaManager.setPeriod(100);

    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

@Test
public void testBatchAll() {
    ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager();
    metaManager.setZkClientx(zkclientx);
    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

/**
 * 查询当前未被ack的batch列表，batchId会按照从小到大进行返回
 */
public List<Long> listBatchIds(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    Map<Long, PositionRange> batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity);
    List<Long> result = new ArrayList<Long>(batchs.keySet());
    Collections.sort(result);
    return result;
}
 

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());

    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }
    synchronized (canalInstance) {
        // 清除batch信息
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
            batchId);
        if (positionRanges == null) { // 说明是重复的ack/rollback
            throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
                clientIdentity.getClientId(),
                batchId));
        }

        // lastRollbackPostions.put(clientIdentity,
        // positionRanges.getEnd());// 记录一下最后rollback的位置
        // TODO 后续rollback到指定的batchId位置
        canalInstance.getEventStore().rollback();// rollback
                                                 // eventStore中的状态信息
        logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
            clientIdentity.getClientId(),
            batchId,
            positionRanges);
    }
}
 

@Test
public void testAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();

    MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
    logPositionManager.start();
    // 构建meta信息
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);

    PositionRange range1 = buildRange(1);
    metaManager.updateCursor(client1, range1.getEnd());

    PositionRange range2 = buildRange(2);
    metaManager.updateCursor(client1, range2.getEnd());

    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);

    PositionRange range3 = buildRange(3);
    metaManager.updateCursor(client2, range3.getEnd());

    PositionRange range4 = buildRange(4);
    metaManager.updateCursor(client2, range4.getEnd());

    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(range2.getEnd(), logPosition);

    metaManager.stop();
    logPositionManager.stop();
}
 

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 

@Test
public void testBatchAll() {
    FileMixedMetaManager metaManager = new FileMixedMetaManager();
    metaManager.setDataDirByFile(dataDir);
    metaManager.setPeriod(100);

    metaManager.start();
    doBatchTest(metaManager);

    metaManager.clearAllBatchs(clientIdentity);
    Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity);
    Assert.assertEquals(0, ranges.size());
    metaManager.stop();
}
 

/**
 * 查询当前未被ack的batch列表，batchId会按照从小到大进行返回
 */
public List<Long> listBatchIds(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    checkSubscribe(clientIdentity);

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    Map<Long, PositionRange> batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity);
    List<Long> result = new ArrayList<Long>(batchs.keySet());
    Collections.sort(result);
    return result;
}
 

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 

public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId)
                                                                                         throws CanalMetaManagerException {
    PositionRange positionRange = super.removeBatch(clientIdentity, batchId);
    // 异步刷新
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.removeBatch(clientIdentity, batchId);
        }
    });

    return positionRange;
}
 

public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId)
                                                                                                                throws CanalMetaManagerException {
    super.addBatch(clientIdentity, positionRange, batchId);
    // 异步刷新
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId);
        }
    });
}