com.alibaba.otter.canal.protocol.ClientIdentity Java Examples

/**
 * 客户端订阅，重复订阅时会更新对应的filter信息
 */
@Override
public void subscribe(ClientIdentity clientIdentity) throws CanalServerException {
    checkStart(clientIdentity.getDestination());

    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    if (!canalInstance.getMetaManager().isStart()) {
        canalInstance.getMetaManager().start();
    }

    canalInstance.getMetaManager().subscribe(clientIdentity); // 执行一下meta订阅

    Position position = canalInstance.getMetaManager().getCursor(clientIdentity);
    if (position == null) {
        position = canalInstance.getEventStore().getFirstPosition();// 获取一下store中的第一条
        if (position != null) {
            canalInstance.getMetaManager().updateCursor(clientIdentity, position); // 更新一下cursor
        }
        logger.info("subscribe successfully, {} with first position:{} ", clientIdentity, position);
    } else {
        logger.info("subscribe successfully, use last cursor position:{} ", clientIdentity, position);
    }

    // 通知下订阅关系变化
    canalInstance.subscribeChange(clientIdentity);
}
 

@Override
public LogPosition getLatestIndexBy(String destination) {
    List<ClientIdentity> clientIdentities = metaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentities)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentities) {
            LogPosition position = (LogPosition) metaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = CanalEventUtils.min(result, position);
            }
        }
    }

    return result;
}
 

private List<ClientIdentity> loadClientIdentity(String destination) {
    List<ClientIdentity> result = Lists.newArrayList();

    FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
    if (data == null) {
        return result;
    }

    List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
    if (clientDatas == null) {
        return result;
    }

    for (FileMetaClientIdentityData clientData : clientDatas) {
        if (clientData.getClientIdentity().getDestination().equals(destination)) {
            result.add(clientData.getClientIdentity());
        }
    }

    return result;
}
 

@Override
public boolean subscribeChange(ClientIdentity identity) {
    if (StringUtils.isNotEmpty(identity.getFilter())) {
        logger.info("subscribe filter change to " + identity.getFilter());
        AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());

        boolean isGroup = (eventParser instanceof GroupEventParser);
        if (isGroup) {
            // 处理group的模式
            List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
                ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
            }
        } else {
            ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
        }

    }

    // filter的处理规则
    // a. parser处理数据过滤处理
    // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
    // 后续内存版的一对多分发，可以考虑
    return true;
}
 

@Test
public void testSubscribeAll() {
    FileMixedMetaManager metaManager = new FileMixedMetaManager();
    metaManager.setDataDirByFile(dataDir);
    metaManager.setPeriod(100);

    metaManager.start();
    doSubscribeTest(metaManager);

    sleep(2000L);
    // 重新构建一次，能获得上一次zk上的记录
    FileMixedMetaManager metaManager2 = new FileMixedMetaManager();
    metaManager2.setDataDirByFile(dataDir);
    metaManager2.setPeriod(100);
    metaManager2.start();

    List<ClientIdentity> clients = metaManager2.listAllSubscribeInfo(destination);
    Assert.assertEquals(2, clients.size());
    metaManager.stop();
}
 

@Test
public void testSubscribeAll() {
    PeriodMixedMetaManager metaManager = new PeriodMixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    doSubscribeTest(metaManager);

    sleep(1000L);
    // 重新构建一次，能获得上一次zk上的记录
    PeriodMixedMetaManager metaManager2 = new PeriodMixedMetaManager();
    metaManager2.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager2.start();

    List<ClientIdentity> clients = metaManager2.listAllSubscribeInfo(destination);
    Assert.assertEquals(2, clients.size());
    metaManager.stop();
}
 

private Position loadCursor(String destination, ClientIdentity clientIdentity) {
    FileMetaInstanceData data = loadDataFromFile(dataFileCaches.get(destination));
    if (data == null) {
        return null;
    }

    List<FileMetaClientIdentityData> clientDatas = data.getClientDatas();
    if (clientDatas == null) {
        return null;
    }

    for (FileMetaClientIdentityData clientData : clientDatas) {
        if (clientData.getClientIdentity() != null && clientData.getClientIdentity().equals(clientIdentity)) {
            return clientData.getCursor();
        }
    }

    return null;
}
 

public void doSubscribeTest(CanalMetaManager metaManager) {
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);
    metaManager.subscribe(client1); // 重复调用
    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);

    List<ClientIdentity> clients = metaManager.listAllSubscribeInfo(destination);
    Assert.assertEquals(Arrays.asList(client1, client2), clients);

    metaManager.unsubscribe(client2);
    ClientIdentity client3 = new ClientIdentity(destination, (short) 3);
    metaManager.subscribe(client3);

    clients = metaManager.listAllSubscribeInfo(destination);
    Assert.assertEquals(Arrays.asList(client1, client3), clients);

}
 

@Override
public List<MetricFamilySamples> collect() {
    List<MetricFamilySamples> mfs = new ArrayList<MetricFamilySamples>();
    GaugeMetricFamily instanceInfo = new GaugeMetricFamily(INSTANCE,
            INSTANCE_HELP, INFO_LABELS_LIST);
    GaugeMetricFamily subsInfo = new GaugeMetricFamily(SUBSCRIPTION,
            SUBSCRIPTION_HELP, DEST_LABELS_LIST);
    for (Map.Entry<String, MetaMetricsHolder> nme : instances.entrySet()) {
        final String destination = nme.getKey();
        final MetaMetricsHolder nmh = nme.getValue();
        instanceInfo.addMetric(nmh.infoLabelValues, 1);
        List<ClientIdentity> subs = nmh.metaManager.listAllSubscribeInfo(destination);
        int count = subs == null ? 0 : subs.size();
        subsInfo.addMetric(nmh.destLabelValues, count);
    }
    mfs.add(instanceInfo);
    mfs.add(subsInfo);
    return mfs;
}
 

public void start() {
    super.start();

    batches = MigrateMap.makeComputingMap(new Function<ClientIdentity, MemoryClientIdentityBatch>() {

        public MemoryClientIdentityBatch apply(ClientIdentity clientIdentity) {
            return MemoryClientIdentityBatch.create(clientIdentity);
        }

    });

    cursors = new MapMaker().makeMap();

    destinations = MigrateMap.makeComputingMap(new Function<String, List<ClientIdentity>>() {

        public List<ClientIdentity> apply(String destination) {
            return Lists.newArrayList();
        }
    });
}
 

/**
 * 找出该destination中可被清理掉的position位置
 * 
 * @param destination
 */
private Position getLatestAckPosition(String destination) {
    List<ClientIdentity> clientIdentitys = canalMetaManager.listAllSubscribeInfo(destination);
    LogPosition result = null;
    if (!CollectionUtils.isEmpty(clientIdentitys)) {
        // 尝试找到一个最小的logPosition
        for (ClientIdentity clientIdentity : clientIdentitys) {
            LogPosition position = (LogPosition) canalMetaManager.getCursor(clientIdentity);
            if (position == null) {
                continue;
            }

            if (result == null) {
                result = position;
            } else {
                result = min(result, position);
            }
        }
    }

    return result;
}
 

public PositionRange getLastestBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long maxBatchId = Collections.max(batchIds);
    PositionRange result = getBatch(clientIdentity, maxBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getLastestBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public PositionRange getFirstBatch(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return null;
    }
    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    PositionRange result = getBatch(clientIdentity, minBatchId);
    if (result == null) { // 出现为null，说明zk节点有变化，重新获取
        return getFirstBatch(clientIdentity);
    } else {
        return result;
    }
}
 

public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}
 

public Map<Long, PositionRange> listAllBatchs(ClientIdentity clientIdentity) {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = null;
    try {
        nodes = zkClientx.getChildren(path);
    } catch (ZkNoNodeException e) {
        // ignore
    }

    if (CollectionUtils.isEmpty(nodes)) {
        return Maps.newHashMap();
    }
    // 找到最大的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }

    Collections.sort(batchIds); // 从小到大排序
    Map<Long, PositionRange> positionRanges = Maps.newLinkedHashMap();
    for (Long batchId : batchIds) {
        PositionRange result = getBatch(clientIdentity, batchId);
        if (result == null) {// 出现为null，说明zk节点有变化，重新获取
            return listAllBatchs(clientIdentity);
        } else {
            positionRanges.put(batchId, result);
        }
    }

    return positionRanges;
}
 

/**
 * 回滚到未进行 {@link #ack} 的地方，下次fetch的时候，可以从最后一个没有 {@link #ack} 的地方开始拿
 */
@Override
public void rollback(ClientIdentity clientIdentity, Long batchId) throws CanalServerException {
    checkStart(clientIdentity.getDestination());
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());

    // 因为存在第一次链接时自动rollback的情况，所以需要忽略未订阅
    boolean hasSubscribe = canalInstance.getMetaManager().hasSubscribe(clientIdentity);
    if (!hasSubscribe) {
        return;
    }
    synchronized (canalInstance) {
        // 清除batch信息
        PositionRange<LogPosition> positionRanges = canalInstance.getMetaManager().removeBatch(clientIdentity,
            batchId);
        if (positionRanges == null) { // 说明是重复的ack/rollback
            throw new CanalServerException(String.format("rollback error, clientId:%s batchId:%d is not exist , please check",
                clientIdentity.getClientId(),
                batchId));
        }

        // lastRollbackPostions.put(clientIdentity,
        // positionRanges.getEnd());// 记录一下最后rollback的位置
        // TODO 后续rollback到指定的batchId位置
        canalInstance.getEventStore().rollback();// rollback
                                                 // eventStore中的状态信息
        logger.info("rollback successfully, clientId:{} batchId:{} position:{}",
            clientIdentity.getClientId(),
            batchId,
            positionRanges);
    }
}
 

public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
    super.subscribe(clientIdentity);

    // 订阅信息频率发生比较低，不需要做定时merge处理
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.subscribe(clientIdentity);
        }
    });
}
 

public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    Position position = super.getCursor(clientIdentity);
    if (position == nullCursor) {
        return null;
    } else {
        return position;
    }
}
 

public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
    super.unsubscribe(clientIdentity);

    // 订阅信息频率发生比较低，不需要做定时merge处理
    executor.submit(new Runnable() {

        public void run() {
            zooKeeperMetaManager.unsubscribe(clientIdentity);
        }
    });
}
 

public void unsubscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
    super.unsubscribe(clientIdentity);

    // 订阅信息频率发生比较低，不需要做定时merge处理
    executor.submit(new Runnable() {

        public void run() {
            flushDataToFile(clientIdentity.getDestination());
        }
    });
}
 

public void subscribe(final ClientIdentity clientIdentity) throws CanalMetaManagerException {
    super.subscribe(clientIdentity);

    // 订阅信息频率发生比较低，不需要做定时merge处理
    executor.submit(new Runnable() {

        public void run() {
            flushDataToFile(clientIdentity.getDestination());
        }
    });
}
 

/**
 * 取消订阅
 */
@Override
public void unsubscribe(ClientIdentity clientIdentity) throws CanalServerException {
    CanalInstance canalInstance = canalInstances.get(clientIdentity.getDestination());
    canalInstance.getMetaManager().unsubscribe(clientIdentity); // 执行一下meta订阅

    logger.info("unsubscribe successfully, {}", clientIdentity);
}
 

public synchronized void subscribe(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    List<ClientIdentity> clientIdentitys = destinations.get(clientIdentity.getDestination());

    if (clientIdentitys.contains(clientIdentity)) {
        clientIdentitys.remove(clientIdentity);
    }

    clientIdentitys.add(clientIdentity);
}
 

public void clearAllBatchs(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> batchChilds = zkClientx.getChildren(path);

    for (String batchChild : batchChilds) {
        String batchPath = path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR + batchChild;
        zkClientx.delete(batchPath);
    }
}
 

@Test
public void testAll() {
    MixedMetaManager metaManager = new MixedMetaManager();

    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);

    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();

    MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
    logPositionManager.start();
    // 构建meta信息
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);

    PositionRange range1 = buildRange(1);
    metaManager.updateCursor(client1, range1.getEnd());

    PositionRange range2 = buildRange(2);
    metaManager.updateCursor(client1, range2.getEnd());

    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);

    PositionRange range3 = buildRange(3);
    metaManager.updateCursor(client2, range3.getEnd());

    PositionRange range4 = buildRange(4);
    metaManager.updateCursor(client2, range4.getEnd());

    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(range2.getEnd(), logPosition);

    metaManager.stop();
    logPositionManager.stop();
}
 

public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}
 

public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());

    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, Position.class);
}
 

public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在，则尝试重建
    }
}
 

public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    String batchPath = zkClientx
        .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
    String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
    return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}
 

public PositionRange removeBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String batchsPath = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    List<String> nodes = zkClientx.getChildren(batchsPath);
    if (CollectionUtils.isEmpty(nodes)) {
        // 没有batch记录
        return null;
    }

    // 找到最小的Id
    ArrayList<Long> batchIds = new ArrayList<Long>(nodes.size());
    for (String batchIdString : nodes) {
        batchIds.add(Long.valueOf(batchIdString));
    }
    Long minBatchId = Collections.min(batchIds);
    if (!minBatchId.equals(batchId)) {
        // 检查一下提交的ack/rollback，必须按batchId分出去的顺序提交，否则容易出现丢数据
        throw new CanalMetaManagerException(String.format("batchId:%d is not the firstly:%d", batchId, minBatchId));
    }

    if (!batchIds.contains(batchId)) {
        // 不存在对应的batchId
        return null;
    }
    PositionRange positionRange = getBatch(clientIdentity, batchId);
    if (positionRange != null) {
        String path = ZookeeperPathUtils
            .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
        zkClientx.delete(path);
    }

    return positionRange;
}