com.alibaba.otter.canal.protocol.position.LogPosition Java Examples

The following examples show how to use com.alibaba.otter.canal.protocol.position.LogPosition. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MixedLogPositionManagerTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);

    MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1000);

    MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 
Example #2
Source File: AbstractCanalStoreScavenge.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 
Example #3
Source File: AbstractCanalStoreScavenge.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
/**
 * 找出一个最小的position位置
 */
private LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库,根据时间进行比较
        if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 
Example #4
Source File: MixedLogPositionManagerTest.java    From canal with Apache License 2.0 6 votes vote down vote up
@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);

    MixedLogPositionManager logPositionManager = new MixedLogPositionManager(zkclientx);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1000);

    MixedLogPositionManager logPositionManager2 = new MixedLogPositionManager(zkclientx);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 
Example #5
Source File: CanalEventUtils.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
/**
 * 找出一个最小的position位置,相等的情况返回position1
 */
public static LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() > position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库,根据时间进行比较
        if (position1.getPostion().getTimestamp() > position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 
Example #6
Source File: GatewayMetaManager.java    From DataLink with Apache License 2.0 6 votes vote down vote up
/**
 * 需要加同步锁,保证在获取最小Position的时候,metaManager是不可变的
 *
 * @throws CanalMetaManagerException
 */
@Override
public synchronized Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    // 入参clientIdentity没有什么作用,因为该方法需要返回所有"Sub Meta Manager"中最小的Position
    List<ClientIdentity> clientIdentities = listAllSubscribeInfo("");
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity item : clientIdentities) {
            LogPosition position = (LogPosition) attachedMetaManagers.get(item.getDestination()).getCursor(item);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 
Example #7
Source File: MetaLogPositionManager.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 
Example #8
Source File: AbstractEventParser.java    From DBus with Apache License 2.0 6 votes vote down vote up
public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 
Example #9
Source File: AbstractEventParser.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 
Example #10
Source File: MemoryEventStoreWithBuffer.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
public Events<Event> get(Position start, int batchSize) throws InterruptedException, CanalStoreException {
    final ReentrantLock lock = this.lock;
    lock.lockInterruptibly();
    try {
        try {
            while (!checkUnGetSlotAt((LogPosition) start, batchSize))
                notEmpty.await();
        } catch (InterruptedException ie) {
            notEmpty.signal(); // propagate to non-interrupted thread
            throw ie;
        }

        return doGet(start, batchSize);
    } finally {
        lock.unlock();
    }
}
 
Example #11
Source File: AbstractCanalStoreScavenge.java    From canal with Apache License 2.0 6 votes vote down vote up
/**
 * 找出一个最小的position位置
 */
private LogPosition min(LogPosition position1, LogPosition position2) {
    if (position1.getIdentity().equals(position2.getIdentity())) {
        // 首先根据文件进行比较
        if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) < 0) {
            return position2;
        } else if (position1.getPostion().getJournalName().compareTo(position2.getPostion().getJournalName()) > 0) {
            return position1;
        } else {
            // 根据offest进行比较
            if (position1.getPostion().getPosition() < position2.getPostion().getPosition()) {
                return position2;
            } else {
                return position1;
            }
        }
    } else {
        // 不同的主备库,根据时间进行比较
        if (position1.getPostion().getTimestamp() < position2.getPostion().getTimestamp()) {
            return position2;
        } else {
            return position1;
        }
    }
}
 
Example #12
Source File: FileMixedLogPositionManagerTest.java    From canal-1.1.3 with Apache License 2.0 6 votes vote down vote up
@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();

    FileMixedLogPositionManager logPositionManager = new FileMixedLogPositionManager(dataDir,
        1000,
        memoryLogPositionManager);
    logPositionManager.start();

    LogPosition position2 = doTest(logPositionManager);
    sleep(1500);

    FileMixedLogPositionManager logPositionManager2 = new FileMixedLogPositionManager(dataDir,
        1000,
        memoryLogPositionManager);
    logPositionManager2.start();

    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);

    logPositionManager.stop();
    logPositionManager2.stop();
}
 
Example #13
Source File: AbstractCanalStoreScavenge.java    From canal with Apache License 2.0 6 votes vote down vote up
/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 
Example #14
Source File: MetaLogPositionManager.java    From canal with Apache License 2.0 6 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 
Example #15
Source File: GatewayInstance.java    From DataLink with Apache License 2.0 5 votes vote down vote up
private void registerInternal() {
    Map<String, Long> timeStamps = new HashMap<>();
    Map<String, RegisterItem> validInstances = new HashMap<>();
    Map<String, RegisterItem> copiedInstances = new HashMap<>(this.endpointInstances);
    copiedInstances.entrySet().forEach(e -> {
        LogPosition position = (LogPosition) e.getValue().instance.getMetaManager()
                .getCursor(new ClientIdentity(e.getKey(), (short) 1001, ""));
        timeStamps.put(e.getKey(), position == null ? System.currentTimeMillis() : position.getPostion().getTimestamp());
    });
    double avgTime = timeStamps.values().stream().mapToLong(p -> p).summaryStatistics().getAverage();
    copiedInstances.entrySet().forEach(e -> {
        //位点时间比平均时间小于两个小时以上时,不参与多路复用
        double diff = avgTime - timeStamps.get(e.getKey());
        if (diff > LESS_THAN_AVG_LOG_TIME_THRESHOLD) {
            kickout(e.getKey(), false,
                    String.format("exceeded threshold than the average log time, this time is [%s] ms, avg time is [%s] ms ," +
                                    " diff time is [%s] ms, threshold is [%s].",
                            timeStamps.get(e.getKey()), avgTime, diff, LESS_THAN_AVG_LOG_TIME_THRESHOLD));
        } else {
            validInstances.put(e.getKey(), e.getValue());
        }
    });


    validInstances.entrySet().forEach(e -> {
        ((GatewayEventSink) workingInstance.getEventSink())
                .registerEventSink(e.getKey(), e.getValue().instance.getEventSink());
        ((GatewayEventStore) workingInstance.getEventStore())
                .registerEventStore(e.getKey(), e.getValue().instance.getEventStore());
        ((GatewayMetaManager) workingInstance.getMetaManager())
                .registerMetaManager(e.getKey(), e.getValue().instance.getMetaManager());
    });
}
 
Example #16
Source File: AbstractLogPositionManagerTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
public LogPosition doTest(CanalLogPositionManager logPositionManager) {
    LogPosition getPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertNull(getPosition);

    LogPosition postion1 = buildPosition(1);
    logPositionManager.persistLogPosition(destination, postion1);
    LogPosition getPosition1 = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(postion1, getPosition1);

    LogPosition postion2 = buildPosition(2);
    logPositionManager.persistLogPosition(destination, postion2);
    LogPosition getPosition2 = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(postion2, getPosition2);
    return postion2;
}
 
Example #17
Source File: CanalReaderMetaManager.java    From DataLink with Apache License 2.0 5 votes vote down vote up
private MysqlReaderPosition buildPosition(Position position) {
    MysqlReaderPosition mysqlReaderPosition = taskPositionManager.translateToMysqlReaderPosition((LogPosition) position);
    CanalReaderEffectSyncPosition effectSyncPosition = canalReader.getEffectSyncPosition();
    TaskShadowInfo taskShadow = canalReader.getTaskShadow();
    if (effectSyncPosition != null) {
        mysqlReaderPosition.setLatestEffectSyncLogFileName(effectSyncPosition.getLatestEffectSyncLogFileName());
        mysqlReaderPosition.setLatestEffectSyncLogFileOffset(effectSyncPosition.getLatestEffectSyncLogFileOffset());
    }
    if (taskShadow != null) {
        mysqlReaderPosition.setShadowId(taskShadow.getId());
    }
    return mysqlReaderPosition;
}
 
Example #18
Source File: MysqlTaskPositionManager.java    From DataLink with Apache License 2.0 5 votes vote down vote up
public LogPosition translateToLogPosition(MysqlReaderPosition cp) {
    LogIdentity logIdentity = new LogIdentity(cp.getSourceAddress(), cp.getSlaveId());

    EntryPosition entryPosition = new EntryPosition(cp.getJournalName(), cp.getPosition(), cp.getTimestamp(), cp.getServerId());
    entryPosition.setIncluded(cp.isIncluded());
    entryPosition.setGtid(cp.getGtid());

    LogPosition logPosition = new LogPosition();
    logPosition.setIdentity(logIdentity);
    logPosition.setPostion(entryPosition);

    return logPosition;
}
 
Example #19
Source File: AbstractEventParser.java    From DBus with Apache License 2.0 5 votes vote down vote up
protected void processSinkError(Throwable e, LogPosition lastPosition, String startBinlogFile, long startPosition) {
    if (lastPosition != null) {
        logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s]",
            lastPosition.getPostion()),
            e);
    } else {
        logger.warn(String.format("ERROR ## parse this event has an error , last position : [%s,%s]",
            startBinlogFile,
            startPosition), e);
    }
}
 
Example #20
Source File: LocalBinlogEventParser.java    From canal with Apache License 2.0 5 votes vote down vote up
@Override
protected EntryPosition findStartPosition(ErosaConnection connection) {
    // 处理逻辑
    // 1. 首先查询上一次解析成功的最后一条记录
    // 2. 存在最后一条记录,判断一下当前记录是否发生过主备切换
    // // a. 无机器切换,直接返回
    // // b. 存在机器切换,按最后一条记录的stamptime进行查找
    // 3. 不存在最后一条记录,则从默认的位置开始启动
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = masterPosition;

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName,尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null) {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        } else {
            if (entryPosition.getPosition() != null) {
                // 如果指定binlogName + offest,直接返回
                return entryPosition;
            } else {
                return new EntryPosition(entryPosition.getTimestamp());
            }
        }
    } else {
        return logPosition.getPostion();
    }

    return null;
}
 
Example #21
Source File: FileMixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
 
Example #22
Source File: CanalServerWithEmbedded.java    From canal with Apache License 2.0 5 votes vote down vote up
/**
 * 回滚到未进行 {@link #ack} 的地方,下次fetch的时候,可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());

    // 因为存在第一次链接时自动rollback的情况,所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }
    synchronized (canalInstance) {
        // 清除batch信息
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
            batchId);
        if (positionRanges == null) { // 说明是重复的ack/rollback
            throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
                clientIdentity.getClientId(),
                batchId));
        }

        // lastRollbackPostions.put(clientIdentity,
        // positionRanges.getEnd());// 记录一下最后rollback的位置
        // TODO 后续rollback到指定的batchId位置
        canalInstance.getEventStore().rollback();// rollback
                                                 // eventStore中的状态信息
        logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
            clientIdentity.getClientId(),
            batchId,
            positionRanges);
    }
}
 
Example #23
Source File: AbstractMetaManagerTest.java    From canal with Apache License 2.0 5 votes vote down vote up
private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 
Example #24
Source File: AbstractEventParser.java    From DBus with Apache License 2.0 5 votes vote down vote up
protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
 
Example #25
Source File: CanalEventUtils.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event, boolean included) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    position.setIncluded(included);

    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}
 
Example #26
Source File: MetaLogPositionManagerTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
@Test
public void testAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();

    MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
    logPositionManager.start();
    // 构建meta信息
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);

    PositionRange range1 = buildRange(1);
    metaManager.updateCursor(client1, range1.getEnd());

    PositionRange range2 = buildRange(2);
    metaManager.updateCursor(client1, range2.getEnd());

    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);

    PositionRange range3 = buildRange(3);
    metaManager.updateCursor(client2, range3.getEnd());

    PositionRange range4 = buildRange(4);
    metaManager.updateCursor(client2, range4.getEnd());

    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(range2.getEnd(), logPosition);

    metaManager.stop();
    logPositionManager.stop();
}
 
Example #27
Source File: MetaLogPositionManagerTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));

    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}
 
Example #28
Source File: MemoryEventStoreWithBuffer.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
/**
 * 检查是否存在需要get的数据,并且数量>=batchSize
 */
private boolean checkUnGetSlotAt(LogPosition startPosition, int batchSize) {
    if (batchMode.isItemSize()) {
        long current = getSequence.get();
        long maxAbleSequence = putSequence.get();
        long next = current;
        if (startPosition == null || !startPosition.getPostion().isIncluded()) { // 第一次订阅之后,需要包含一下start位置,防止丢失第一条记录
            next = next + 1;// 少一条数据
        }

        if (current < maxAbleSequence && next + batchSize - 1 <= maxAbleSequence) {
            return true;
        } else {
            return false;
        }
    } else {
        // 处理内存大小判断
        long currentSize = getMemSize.get();
        long maxAbleSize = putMemSize.get();

        if (maxAbleSize - currentSize >= batchSize * bufferMemUnit) {
            return true;
        } else {
            return false;
        }
    }
}
 
Example #29
Source File: ZooKeeperLogPositionManager.java    From canal with Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
 
Example #30
Source File: FileMixedLogPositionManager.java    From canal with Apache License 2.0 5 votes vote down vote up
private void flushDataToFile(String destination, File dataFile) {
    LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
    if (position != null && position != nullPosition) {
        String json = JsonUtils.marshalToString(position);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}