com.alibaba.otter.canal.protocol.position.PositionRange Java Examples
The following examples show how to use
com.alibaba.otter.canal.protocol.position.PositionRange.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PeriodMixedMetaManagerTest.java From canal with Apache License 2.0 | 6 votes |
@Test public void testBatchAll() { PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager(); ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(zkclientx); metaManager.setZooKeeperMetaManager(zooKeeperMetaManager); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #2
Source File: PeriodMixedMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
@Test public void testBatchAll() { PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager(); ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(zkclientx); metaManager.setZooKeeperMetaManager(zooKeeperMetaManager); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #3
Source File: AbstractMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
public Position doCursorTest(CanalMetaManager metaManager) { metaManager.subscribe(clientIdentity); Position position1 = metaManager.getCursor(clientIdentity); Assert.assertNull(position1); PositionRange range = buildRange(1); metaManager.updateCursor(clientIdentity, range.getStart()); Position position2 = metaManager.getCursor(clientIdentity); Assert.assertEquals(range.getStart(), position2); metaManager.updateCursor(clientIdentity, range.getEnd()); Position position3 = metaManager.getCursor(clientIdentity); Assert.assertEquals(range.getEnd(), position3); return position3; }
Example #4
Source File: MixedMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
@Test public void testBatchAll() { MixedMetaManager metaManager = new MixedMetaManager(); ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(zkclientx); metaManager.setZooKeeperMetaManager(zooKeeperMetaManager); metaManager.start(); doBatchTest(metaManager); sleep(1000L); // 重新构建一次,能获得上一次zk上的记录 MixedMetaManager metaManager2 = new MixedMetaManager(); metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager); metaManager2.start(); Map<Long, PositionRange> ranges = metaManager2.listAllBatchs(clientIdentity); Assert.assertEquals(3, ranges.size()); metaManager.clearAllBatchs(clientIdentity); ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); metaManager2.stop(); }
Example #5
Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
public PositionRange getFirstBatch(ClientIdentity clientIdentity) { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); List<String> nodes = null; try { nodes = zkClientx.getChildren(path); } catch (ZkNoNodeException e) { // ignore } if (CollectionUtils.isEmpty(nodes)) { return null; } // 找到最小的Id ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); for (String batchIdString : nodes) { batchIds.add(Long.valueOf(batchIdString)); } Long minBatchId = Collections.min(batchIds); PositionRange result = getBatch(clientIdentity, minBatchId); if (result == null) { // 出现为null,说明zk节点有变化,重新获取 return getFirstBatch(clientIdentity); } else { return result; } }
Example #6
Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
public PositionRange getLastestBatch(ClientIdentity clientIdentity) { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); List<String> nodes = null; try { nodes = zkClientx.getChildren(path); } catch (ZkNoNodeException e) { // ignore } if (CollectionUtils.isEmpty(nodes)) { return null; } // 找到最大的Id ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); for (String batchIdString : nodes) { batchIds.add(Long.valueOf(batchIdString)); } Long maxBatchId = Collections.max(batchIds); PositionRange result = getBatch(clientIdentity, maxBatchId); if (result == null) { // 出现为null,说明zk节点有变化,重新获取 return getLastestBatch(clientIdentity); } else { return result; } }
Example #7
Source File: AbstractMetaManagerTest.java From canal with Apache License 2.0 | 6 votes |
public Position doCursorTest(CanalMetaManager metaManager) { metaManager.subscribe(clientIdentity); Position position1 = metaManager.getCursor(clientIdentity); Assert.assertNull(position1); PositionRange range = buildRange(1); metaManager.updateCursor(clientIdentity, range.getStart()); Position position2 = metaManager.getCursor(clientIdentity); Assert.assertEquals(range.getStart(), position2); metaManager.updateCursor(clientIdentity, range.getEnd()); Position position3 = metaManager.getCursor(clientIdentity); Assert.assertEquals(range.getEnd(), position3); return position3; }
Example #8
Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0 | 6 votes |
public PositionRange getFirstBatch(ClientIdentity clientIdentity) { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); List<String> nodes = null; try { nodes = zkClientx.getChildren(path); } catch (ZkNoNodeException e) { // ignore } if (CollectionUtils.isEmpty(nodes)) { return null; } // 找到最小的Id ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); for (String batchIdString : nodes) { batchIds.add(Long.valueOf(batchIdString)); } Long minBatchId = Collections.min(batchIds); PositionRange result = getBatch(clientIdentity, minBatchId); if (result == null) { // 出现为null,说明zk节点有变化,重新获取 return getFirstBatch(clientIdentity); } else { return result; } }
Example #9
Source File: ZooKeeperMetaManager.java From canal with Apache License 2.0 | 6 votes |
public PositionRange getLastestBatch(ClientIdentity clientIdentity) { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); List<String> nodes = null; try { nodes = zkClientx.getChildren(path); } catch (ZkNoNodeException e) { // ignore } if (CollectionUtils.isEmpty(nodes)) { return null; } // 找到最大的Id ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); for (String batchIdString : nodes) { batchIds.add(Long.valueOf(batchIdString)); } Long maxBatchId = Collections.max(batchIds); PositionRange result = getBatch(clientIdentity, maxBatchId); if (result == null) { // 出现为null,说明zk节点有变化,重新获取 return getLastestBatch(clientIdentity); } else { return result; } }
Example #10
Source File: MemoryMetaManager.java From canal with Apache License 2.0 | 5 votes |
public synchronized PositionRange getFirstPositionRange() { if (batches.size() == 0) { return null; } else { Long batchId = Collections.min(batches.keySet()); return batches.get(batchId); } }
Example #11
Source File: MemoryMetaManager.java From canal with Apache License 2.0 | 5 votes |
public synchronized PositionRange removePositionRange(Long batchId) { if (batches.containsKey(batchId)) { Long minBatchId = Collections.min(batches.keySet()); if (!minBatchId.equals(batchId)) { // 检查一下提交的ack/rollback,必须按batchId分出去的顺序提交,否则容易出现丢数据 throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId)); } return batches.remove(batchId); } else { return null; } }
Example #12
Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException { String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); List<String> nodes = zkClientx.getChildren(batchsPath); if (CollectionUtils.isEmpty(nodes)) { // 没有batch记录 return null; } // 找到最小的Id ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); for (String batchIdString : nodes) { batchIds.add(Long.valueOf(batchIdString)); } Long minBatchId = Collections.min(batchIds); if (!minBatchId.equals(batchId)) { // 检查一下提交的ack/rollback,必须按batchId分出去的顺序提交,否则容易出现丢数据 throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId)); } if (!batchIds.contains(batchId)) { // 不存在对应的batchId return null; } PositionRange positionRange = getBatch(clientIdentity, batchId); if (positionRange != null) { String path = ZookeeperPathUtils .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId); zkClientx.delete(path); } return positionRange; }
Example #13
Source File: MemoryMetaManager.java From canal with Apache License 2.0 | 5 votes |
public synchronized PositionRange getLastestPositionRange() { if (batches.size() == 0) { return null; } else { Long batchId = Collections.max(batches.keySet()); return batches.get(batchId); } }
Example #14
Source File: MemoryMetaManagerTest.java From canal with Apache License 2.0 | 5 votes |
@Test public void testBatchAll() { MemoryMetaManager metaManager = new MemoryMetaManager(); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #15
Source File: ZooKeeperMetaManager.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) { String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(), clientIdentity.getClientId()); List<String> nodes = null; try { nodes = zkClientx.getChildren(path); } catch (ZkNoNodeException e) { // ignore } if (CollectionUtils.isEmpty(nodes)) { return Maps.newHashMap(); } // 找到最大的Id ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size()); for (String batchIdString : nodes) { batchIds.add(Long.valueOf(batchIdString)); } Collections.sort(batchIds); // 从小到大排序 Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap(); for (Long batchId : batchIds) { PositionRange result = getBatch(clientIdentity, batchId); if (result == null) {// 出现为null,说明zk节点有变化,重新获取 return listAllBatchs(clientIdentity); } else { positionRanges.put(batchId, result); } } return positionRanges; }
Example #16
Source File: MetaLogPositionManagerTest.java From canal with Apache License 2.0 | 5 votes |
@Test public void testAll() { MixedMetaManager metaManager = new MixedMetaManager(); ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(zkclientx); metaManager.setZooKeeperMetaManager(zooKeeperMetaManager); metaManager.start(); MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager); logPositionManager.start(); // 构建meta信息 ClientIdentity client1 = new ClientIdentity(destination, (short) 1); metaManager.subscribe(client1); PositionRange range1 = buildRange(1); metaManager.updateCursor(client1, range1.getEnd()); PositionRange range2 = buildRange(2); metaManager.updateCursor(client1, range2.getEnd()); ClientIdentity client2 = new ClientIdentity(destination, (short) 2); metaManager.subscribe(client2); PositionRange range3 = buildRange(3); metaManager.updateCursor(client2, range3.getEnd()); PositionRange range4 = buildRange(4); metaManager.updateCursor(client2, range4.getEnd()); LogPosition logPosition = logPositionManager.getLatestIndexBy(destination); Assert.assertEquals(range2.getEnd(), logPosition); metaManager.stop(); logPositionManager.stop(); }
Example #17
Source File: MixedMetaManager.java From canal with Apache License 2.0 | 5 votes |
public Long addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange) throws CanalMetaManagerException { final Long batchId = super.addBatch(clientIdentity, positionRange); // 异步刷新 executor.submit(new Runnable() { public void run() { zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId); } }); return batchId; }
Example #18
Source File: AbstractMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
private PositionRange<LogPosition> buildRange(int number) { LogPosition start = new LogPosition(); start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime())); LogPosition end = new LogPosition(); end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L)); return new PositionRange<LogPosition>(start, end); }
Example #19
Source File: MemoryMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
@Test public void testBatchAll() { MemoryMetaManager metaManager = new MemoryMetaManager(); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #20
Source File: FileMixedMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
@Test public void testBatchAll() { FileMixedMetaManager metaManager = new FileMixedMetaManager(); metaManager.setDataDirByFile(dataDir); metaManager.setPeriod(100); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #21
Source File: ZooKeeperMetaManagerTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
@Test public void testBatchAll() { ZooKeeperMetaManager metaManager = new ZooKeeperMetaManager(); metaManager.setZkClientx(zkclientx); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #22
Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
/** * 查询当前未被ack的batch列表,batchId会按照从小到大进行返回 */ public List<Long> listBatchIds(ClientIdentity clientIdentity) throws CanalServerException { checkStart(clientIdentity.getDestination()); checkSubscribe(clientIdentity); CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination()); Map<Long, PositionRange> batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity); List<Long> result = new ArrayList<Long>(batchs.keySet()); Collections.sort(result); return result; }
Example #23
Source File: CanalServerWithEmbedded.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
/** * 回滚到未进行 {@link #ack} 的地方,下次fetch的时候,可以从最后一个没有 {@link #ack} 的地方开始拿 */ @Override public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException { checkStart(clientIdentity.getDestination()); CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination()); // 因为存在第一次链接时自动rollback的情况,所以需要忽略未订阅 boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity); if (!hasSubscribe) { return; } synchronized (canalInstance) { // 清除batch信息 PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity, batchId); if (positionRanges == null) { // 说明是重复的ack/rollback throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check", clientIdentity.getClientId(), batchId)); } // lastRollbackPostions.put(clientIdentity, // positionRanges.getEnd());// 记录一下最后rollback的位置 // TODO 后续rollback到指定的batchId位置 canalInstance.getEventStore().rollback();// rollback // eventStore中的状态信息 logger.info("rollback successfully, clientId:{} batchId:{} position:{}", clientIdentity.getClientId(), batchId, positionRanges); } }
Example #24
Source File: MetaLogPositionManagerTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
@Test public void testAll() { MixedMetaManager metaManager = new MixedMetaManager(); ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager(); zooKeeperMetaManager.setZkClientx(zkclientx); metaManager.setZooKeeperMetaManager(zooKeeperMetaManager); metaManager.start(); MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager); logPositionManager.start(); // 构建meta信息 ClientIdentity client1 = new ClientIdentity(destination, (short) 1); metaManager.subscribe(client1); PositionRange range1 = buildRange(1); metaManager.updateCursor(client1, range1.getEnd()); PositionRange range2 = buildRange(2); metaManager.updateCursor(client1, range2.getEnd()); ClientIdentity client2 = new ClientIdentity(destination, (short) 2); metaManager.subscribe(client2); PositionRange range3 = buildRange(3); metaManager.updateCursor(client2, range3.getEnd()); PositionRange range4 = buildRange(4); metaManager.updateCursor(client2, range4.getEnd()); LogPosition logPosition = logPositionManager.getLatestIndexBy(destination); Assert.assertEquals(range2.getEnd(), logPosition); metaManager.stop(); logPositionManager.stop(); }
Example #25
Source File: MetaLogPositionManagerTest.java From canal-1.1.3 with Apache License 2.0 | 5 votes |
private PositionRange<LogPosition> buildRange(int number) { LogPosition start = new LogPosition(); start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime())); LogPosition end = new LogPosition(); end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L)); return new PositionRange<LogPosition>(start, end); }
Example #26
Source File: FileMixedMetaManagerTest.java From canal with Apache License 2.0 | 5 votes |
@Test public void testBatchAll() { FileMixedMetaManager metaManager = new FileMixedMetaManager(); metaManager.setDataDirByFile(dataDir); metaManager.setPeriod(100); metaManager.start(); doBatchTest(metaManager); metaManager.clearAllBatchs(clientIdentity); Map<Long, PositionRange> ranges = metaManager.listAllBatchs(clientIdentity); Assert.assertEquals(0, ranges.size()); metaManager.stop(); }
Example #27
Source File: CanalServerWithEmbedded.java From canal with Apache License 2.0 | 5 votes |
/** * 查询当前未被ack的batch列表,batchId会按照从小到大进行返回 */ public List<Long> listBatchIds(ClientIdentity clientIdentity) throws CanalServerException { checkStart(clientIdentity.getDestination()); checkSubscribe(clientIdentity); CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination()); Map<Long, PositionRange> batchs = canalInstance.getMetaManager().listAllBatchs(clientIdentity); List<Long> result = new ArrayList<Long>(batchs.keySet()); Collections.sort(result); return result; }
Example #28
Source File: AbstractMetaManagerTest.java From canal with Apache License 2.0 | 5 votes |
private PositionRange<LogPosition> buildRange(int number) { LogPosition start = new LogPosition(); start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime())); LogPosition end = new LogPosition(); end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L)); end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L)); return new PositionRange<LogPosition>(start, end); }
Example #29
Source File: MixedMetaManager.java From canal with Apache License 2.0 | 5 votes |
public PositionRange removeBatch(final ClientIdentity clientIdentity, final Long batchId) throws CanalMetaManagerException { PositionRange positionRange = super.removeBatch(clientIdentity, batchId); // 异步刷新 executor.submit(new Runnable() { public void run() { zooKeeperMetaManager.removeBatch(clientIdentity, batchId); } }); return positionRange; }
Example #30
Source File: MixedMetaManager.java From canal with Apache License 2.0 | 5 votes |
public void addBatch(final ClientIdentity clientIdentity, final PositionRange positionRange, final Long batchId) throws CanalMetaManagerException { super.addBatch(clientIdentity, positionRange, batchId); // 异步刷新 executor.submit(new Runnable() { public void run() { zooKeeperMetaManager.addBatch(clientIdentity, positionRange, batchId); } }); }