com.alibaba.otter.canal.common.utils.JsonUtils Java Examples

The following examples show how to use com.alibaba.otter.canal.common.utils.JsonUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ZooKeeperMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());

    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, Position.class);
}
 
Example #2
Source File: FileMixedLogPositionManager.java    From canal with Apache License 2.0 5 votes vote down vote up
private LogPosition loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, LogPosition.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}
 
Example #3
Source File: FileMixedLogPositionManager.java    From canal with Apache License 2.0 5 votes vote down vote up
private void flushDataToFile(String destination, File dataFile) {
    LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
    if (position != null && position != nullPosition) {
        String json = JsonUtils.marshalToString(position);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
 
Example #4
Source File: ZooKeeperLogPositionManager.java    From canal with Apache License 2.0 5 votes vote down vote up
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}
 
Example #5
Source File: ZooKeeperLogPositionManager.java    From canal with Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
 
Example #6
Source File: ClusterNodeAccessStrategy.java    From canal with Apache License 2.0 5 votes vote down vote up
private void initRunning(Object data) {
    if (data == null) {
        return;
    }

    ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
    String[] strs = StringUtils.split(runningData.getAddress(), ':');
    if (strs.length == 2) {
        runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
    }
}
 
Example #7
Source File: ClientRunningMonitor.java    From canal with Apache License 2.0 5 votes vote down vote up
private void processActiveEnter() {
    if (listener != null) {
        // 触发回调,建立与server的socket链接
        InetSocketAddress connectAddress = listener.processActiveEnter();
        String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
        this.clientData.setAddress(address);

        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
            this.clientData.getClientId());
        // 序列化
        byte[] bytes = JsonUtils.marshalToByte(clientData);
        zkClient.writeData(path, bytes);
    }
}
 
Example #8
Source File: FileMixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
 
Example #9
Source File: FileMixedMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
private FileMetaInstanceData loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}
 
Example #10
Source File: ZooKeeperMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}
 
Example #11
Source File: ZooKeeperMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
                     Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    zkClientx.createPersistent(path, data, true);
}
 
Example #12
Source File: ZooKeeperMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    String batchPath = zkClientx
        .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
    String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
    return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}
 
Example #13
Source File: ZooKeeperMetaManager.java    From canal with Apache License 2.0 5 votes vote down vote up
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在,则尝试重建
    }
}
 
Example #14
Source File: ZooKeeperMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
public Position getCursor(ClientIdentity clientIdentity) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());

    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, Position.class);
}
 
Example #15
Source File: FileMixedLogPositionManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private LogPosition loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, LogPosition.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}
 
Example #16
Source File: FileMixedLogPositionManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private void flushDataToFile(String destination, File dataFile) {
    LogPosition position = memoryLogPositionManager.getLatestIndexBy(destination);
    if (position != null && position != nullPosition) {
        String json = JsonUtils.marshalToString(position);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
 
Example #17
Source File: ZooKeeperLogPositionManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
@Override
public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = JsonUtils.marshalToByte(logPosition);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);
    }
}
 
Example #18
Source File: ZooKeeperLogPositionManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
@Override
public LogPosition getLatestIndexBy(String destination) {
    String path = ZookeeperPathUtils.getParsePath(destination);
    byte[] data = zkClientx.readData(path, true);
    if (data == null || data.length == 0) {
        return null;
    }

    return JsonUtils.unmarshalFromByte(data, LogPosition.class);
}
 
Example #19
Source File: ClusterNodeAccessStrategy.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private void initRunning(Object data) {
    if (data == null) {
        return;
    }

    ServerRunningData runningData = JsonUtils.unmarshalFromByte((byte[]) data, ServerRunningData.class);
    String[] strs = StringUtils.split(runningData.getAddress(), ':');
    if (strs.length == 2) {
        runningAddress = new InetSocketAddress(strs[0], Integer.valueOf(strs[1]));
    }
}
 
Example #20
Source File: ClientRunningMonitor.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private void processActiveEnter() {
    if (listener != null) {
        // 触发回调,建立与server的socket链接
        InetSocketAddress connectAddress = listener.processActiveEnter();
        String address = connectAddress.getAddress().getHostAddress() + ":" + connectAddress.getPort();
        this.clientData.setAddress(address);

        String path = ZookeeperPathUtils.getDestinationClientRunning(this.destination,
            this.clientData.getClientId());
        // 序列化
        byte[] bytes = JsonUtils.marshalToByte(clientData);
        zkClient.writeData(path, bytes);
    }
}
 
Example #21
Source File: FileMixedMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private void flushDataToFile(String destination, File dataFile) {
    FileMetaInstanceData data = new FileMetaInstanceData();
    if (destinations.containsKey(destination)) {
        synchronized (destination.intern()) { // 基于destination控制一下并发更新
            data.setDestination(destination);

            List<FileMetaClientIdentityData> clientDatas = Lists.newArrayList();
            List<ClientIdentity> clientIdentitys = destinations.get(destination);
            for (ClientIdentity clientIdentity : clientIdentitys) {
                FileMetaClientIdentityData clientData = new FileMetaClientIdentityData();
                clientData.setClientIdentity(clientIdentity);
                Position position = cursors.get(clientIdentity);
                if (position != null && position != nullCursor) {
                    clientData.setCursor((LogPosition) position);
                }

                clientDatas.add(clientData);
            }

            data.setClientDatas(clientDatas);
        }

        String json = JsonUtils.marshalToString(data);
        try {
            FileUtils.writeStringToFile(dataFile, json);
        } catch (IOException e) {
            throw new CanalMetaManagerException(e);
        }
    }
}
 
Example #22
Source File: FileMixedMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
private FileMetaInstanceData loadDataFromFile(File dataFile) {
    try {
        if (!dataFile.exists()) {
            return null;
        }

        String json = FileUtils.readFileToString(dataFile, charset.name());
        return JsonUtils.unmarshalFromString(json, FileMetaInstanceData.class);
    } catch (IOException e) {
        throw new CanalMetaManagerException(e);
    }
}
 
Example #23
Source File: ZooKeeperMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
public PositionRange getBatch(ClientIdentity clientIdentity, Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = zkClientx.readData(path, true);
    if (data == null) {
        return null;
    }

    PositionRange positionRange = JsonUtils.unmarshalFromByte(data, PositionRange.class);
    return positionRange;
}
 
Example #24
Source File: ZooKeeperMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
public void addBatch(ClientIdentity clientIdentity, PositionRange positionRange,
                     Long batchId) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils
        .getBatchMarkWithIdPath(clientIdentity.getDestination(), clientIdentity.getClientId(), batchId);
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    zkClientx.createPersistent(path, data, true);
}
 
Example #25
Source File: ZooKeeperMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
public Long addBatch(ClientIdentity clientIdentity, PositionRange positionRange) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getBatchMarkPath(clientIdentity.getDestination(),
        clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(positionRange, SerializerFeature.WriteClassName);
    String batchPath = zkClientx
        .createPersistentSequential(path + ZookeeperPathUtils.ZOOKEEPER_SEPARATOR, data, true);
    String batchIdString = StringUtils.substringAfterLast(batchPath, ZookeeperPathUtils.ZOOKEEPER_SEPARATOR);
    return ZookeeperPathUtils.getBatchMarkId(batchIdString);
}
 
Example #26
Source File: ZooKeeperMetaManager.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up
public void updateCursor(ClientIdentity clientIdentity, Position position) throws CanalMetaManagerException {
    String path = ZookeeperPathUtils.getCursorPath(clientIdentity.getDestination(), clientIdentity.getClientId());
    byte[] data = JsonUtils.marshalToByte(position, SerializerFeature.WriteClassName);
    try {
        zkClientx.writeData(path, data);
    } catch (ZkNoNodeException e) {
        zkClientx.createPersistent(path, data, true);// 第一次节点不存在,则尝试重建
    }
}
 
Example #27
Source File: MysqlEventParser.java    From canal with Apache License 2.0 4 votes vote down vote up
protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {// 找不到历史成功记录
        EntryPosition entryPosition = null;
        if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
            entryPosition = masterPosition;
        } else if (standbyInfo != null
                   && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
            entryPosition = standbyPosition;
        }

        if (entryPosition == null) {
            entryPosition = findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
        }

        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName,尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { "", "", entryPosition.getTimestamp() });
                return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
            } else {
                logger.warn("prepare to find start position just show master status");
                return findEndPositionWithMasterIdAndTimestamp(mysqlConnection); // 默认从当前最后一个位置进行消费
            }
        } else {
            if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                // 如果指定binlogName + offest,直接返回
                entryPosition = findPositionWithMasterIdAndTimestamp(mysqlConnection, entryPosition);
                logger.warn("prepare to find start position {}:{}:{}",
                    new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(),
                            entryPosition.getTimestamp() });
                return entryPosition;
            } else {
                EntryPosition specificLogFilePosition = null;
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    // 如果指定binlogName +
                    // timestamp,但没有指定对应的offest,尝试根据时间找一下offest
                    EntryPosition endPosition = findEndPosition(mysqlConnection);
                    if (endPosition != null) {
                        logger.warn("prepare to find start position {}:{}:{}",
                            new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
                        specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection,
                            entryPosition.getTimestamp(),
                            endPosition,
                            entryPosition.getJournalName(),
                            true);
                    }
                }

                if (specificLogFilePosition == null) {
                    // position不存在,从文件头开始
                    entryPosition.setPosition(BINLOG_START_OFFEST);
                    return entryPosition;
                } else {
                    return specificLogFilePosition;
                }
            }
        }
    } else {
        if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
            if (dumpErrorCountThreshold >= 0 && dumpErrorCount > dumpErrorCountThreshold) {
                // binlog定位位点失败,可能有两个原因:
                // 1. binlog位点被删除
                // 2.vip模式的mysql,发生了主备切换,判断一下serverId是否变化,针对这种模式可以发起一次基于时间戳查找合适的binlog位点
                boolean case2 = (standbyInfo == null || standbyInfo.getAddress() == null)
                                && logPosition.getPostion().getServerId() != null
                                && !logPosition.getPostion().getServerId().equals(findServerId(mysqlConnection));
                if (case2) {
                    EntryPosition findPosition = fallbackFindByStartTimestamp(logPosition, mysqlConnection);
                    dumpErrorCount = 0;
                    return findPosition;
                }

                Long timestamp = logPosition.getPostion().getTimestamp();
                if (isRdsOssMode() && (timestamp != null && timestamp > 0)) {
                    // 如果binlog位点不存在,并且属于timestamp不为空,可以返回null走到oss binlog处理
                    return null;
                }
            } else if (StringUtils.isBlank(logPosition.getPostion().getJournalName())
                    && logPosition.getPostion().getPosition() <= 0
                    && logPosition.getPostion().getTimestamp() > 0) {
                return fallbackFindByStartTimestamp(logPosition,mysqlConnection);
            }
            // 其余情况
            logger.warn("prepare to find start position just last position\n {}",
                JsonUtils.marshalToString(logPosition));
            return logPosition.getPostion();
        } else {
            // 针对切换的情况,考虑回退时间
            long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
            logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "",
                    logPosition.getPostion().getTimestamp() });
            return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
        }
    }
}