com.alibaba.otter.canal.instance.manager.model.CanalParameter Java Examples

The following examples show how to use com.alibaba.otter.canal.instance.manager.model.CanalParameter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CanalServerWithEmbedded_StandaloneTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
protected Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("test");

    CanalParameter parameter = new CanalParameter();

    parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
    parameter.setMetaMode(MetaMode.MEMORY);
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.MEMORY);

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000003\",\"position\":14217L,\"timestamp\":\"1505998863000\"}",
        "{\"journalName\":\"mysql-bin.000003\",\"position\":14377L,\"timestamp\":\"1505998863000\"}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #2
Source File: CanalServerWithEmbedded_StandbyTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
protected Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("test");

    CanalParameter parameter = new CanalParameter();

    parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
    parameter.setMetaMode(MetaMode.MIXED); // 冷备，可选择混合模式
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.META);// 内存版store，需要选择meta做为index

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
        "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #3
Source File: CanalServerWithNettyTest.java    From canal-1.1.3 with Apache License 2.0 5 votes vote down vote up 
private Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("test");

    CanalParameter parameter = new CanalParameter();

    parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
    parameter.setMetaMode(MetaMode.MEMORY);
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.MEMORY);

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
        "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #4
Source File: CanalServerWithEmbedded_StandaloneTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("test");

    CanalParameter parameter = new CanalParameter();

    parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
    parameter.setMetaMode(MetaMode.MEMORY);
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.MEMORY);

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000003\",\"position\":14217L,\"timestamp\":\"1505998863000\"}",
        "{\"journalName\":\"mysql-bin.000003\",\"position\":14377L,\"timestamp\":\"1505998863000\"}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #5
Source File: CanalServerWithEmbedded_StandbyTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
protected Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("test");

    CanalParameter parameter = new CanalParameter();

    parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
    parameter.setMetaMode(MetaMode.MIXED); // 冷备，可选择混合模式
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.META);// 内存版store，需要选择meta做为index

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
        "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #6
Source File: CanalServerTest.java    From canal with Apache License 2.0 5 votes vote down vote up 
private Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("test");

    CanalParameter parameter = new CanalParameter();

    parameter.setZkClusters(Arrays.asList("127.0.0.1:2188"));
    parameter.setMetaMode(MetaMode.MEMORY);
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.MEMORY);

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}",
        "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #7
Source File: CanalServerWithEmbedded_FileModeTest.java    From canal-1.1.3 with Apache License 2.0 4 votes vote down vote up 
protected Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("my standalone server test ");

    CanalParameter parameter = new CanalParameter();

    parameter.setMetaMode(MetaMode.LOCAL_FILE);
    parameter.setDataDir("./conf");
    parameter.setMetaFileFlushPeriod(1000);
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.MEMORY_META_FAILBACK);

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}",
        "{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}
Example #8
Source File: CustomCanalInstanceWithManager.java    From DataLink with Apache License 2.0 4 votes vote down vote up 
@Override
protected void initEventParser() {
    logger.info("init eventParser begin...");
    CanalParameter.SourcingType type = parameters.getSourcingType();

    List<List<CanalParameter.DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
    if (!CollectionUtils.isEmpty(groupDbAddresses)) {
        int size = groupDbAddresses.get(0).size();// 取第一个分组的数量，主备分组的数量必须一致
        List<CanalEventParser> eventParsers = new ArrayList<CanalEventParser>();
        for (int i = 0; i < size; i++) {
            List<InetSocketAddress> dbAddress = new ArrayList<InetSocketAddress>();
            CanalParameter.SourcingType lastType = null;
            for (List<CanalParameter.DataSourcing> groupDbAddress : groupDbAddresses) {
                if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
                    throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s",
                            lastType,
                            groupDbAddress.get(i).getType()));
                }

                lastType = groupDbAddress.get(i).getType();
                dbAddress.add(groupDbAddress.get(i).getDbAddress());
            }

            // 初始化其中的一个分组parser
            eventParsers.add(doInitEventParser(lastType, dbAddress, (size > 1) ? true : false));//@@changes
        }

        if (eventParsers.size() > 1) { // 如果存在分组，构造分组的parser
            GroupEventParser groupEventParser = new GroupEventParser();
            groupEventParser.setEventParsers(eventParsers);
            this.eventParser = groupEventParser;
        } else {
            this.eventParser = eventParsers.get(0);
        }
    } else {
        // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
        this.eventParser = doInitEventParser(type, new ArrayList<InetSocketAddress>(), false);//@@changes
    }

    logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}
Example #9
Source File: GatewayUtil.java    From DataLink with Apache License 2.0 4 votes vote down vote up 
private static String buildCanalName(Canal canal) {
    List<List<CanalParameter.DataSourcing>> datasources = canal.getCanalParameter().getGroupDbAddresses();
    CanalParameter.DataSourcing master = datasources.get(0).get(0);
    InetSocketAddress socketAddress = master.getDbAddress();
    return NAME_PREFIX + "-" + socketAddress.getHostName() + "-" + socketAddress.getPort();
}
Example #10
Source File: CanalReaderConfigGenerator.java    From DataLink with Apache License 2.0 4 votes vote down vote up 
@SuppressWarnings("unchecked")
private static CanalParameter buildCanalParameter(String destination, MysqlReaderParameter parameter, TaskReaderContext context,
                                                  CanalReaderType readerType, TaskShadowInfo taskShadow) {
    CanalParameter canalParameter = new CanalParameter();

    //zookeeper相关信息不在此配置，由使用者按需进行设置
    //此处只是为了进行显示说明的需要，进行了赋值为null的操作（正常情况下没必要，因为默认值就为null）
    canalParameter.setZkClusterId(null);
    canalParameter.setZkClusters(null);

    //数据库配置，页面上直接选数据库，然后在此处进行组装
    MediaSourceInfo mediaSrc = context.getService(MediaService.class).getMediaSourceById(parameter.getMediaSourceId());
    //组装数据库用户名、密码等
    packCanalParameter(destination, canalParameter, mediaSrc, parameter, context);

    //绝大部分情况下只指定Timestamps就可满足需求
    if (readerType == CanalReaderType.MAIN) {
        canalParameter.setPositions(
                Lists.newArrayList(
                        JSON.toJSONString(new EntryPosition(parameter.getStartTimeStamps())),
                        JSON.toJSONString(new EntryPosition(parameter.getStartTimeStamps()))
                ));
    } else {
        if (taskShadow == null) {
            throw new IllegalStateException("readerType is SHADOW , but taskShadow is null.");
        }
        canalParameter.setPositions(
                Lists.newArrayList(
                        JSON.toJSONString(new EntryPosition(taskShadow.getParameterObj().getTimeStamp())),
                        JSON.toJSONString(new EntryPosition(taskShadow.getParameterObj().getTimeStamp()))
                ));
    }

    //位点配置考虑多个库的情况
    canalParameter.setStorageMode(CanalParameter.StorageMode.MEMORY);//限定为MEMORY,有需要时再改为可支持其它模式
    canalParameter.setStorageBatchMode(CanalParameter.BatchMode.MEMSIZE);//限定为MEMORY,有需要时再改为可支持其它模式
    canalParameter.setMemoryStorageBufferSize(parameter.getMemoryStorageBufferSize());
    canalParameter.setMemoryStorageBufferMemUnit(parameter.getMemoryStorageBufferMemUnit());
    canalParameter.setHaMode(CanalParameter.HAMode.HEARTBEAT);//限定为HEARTBEAT，有需要时再改为可支持其它模式
    canalParameter.setDetectingEnable(true);
    canalParameter.setHeartbeatHaEnable(true);
    canalParameter.setDetectingSQL(parameter.getDetectingSQL());
    canalParameter.setDetectingIntervalInSeconds(parameter.getDetectingIntervalInSeconds());
    canalParameter.setDetectingRetryTimes(parameter.getDetectingRetryTimes());
    canalParameter.setDetectingTimeoutThresholdInSeconds(parameter.getDetectingTimeoutThresholdInSeconds());
    canalParameter.setMetaMode(CanalParameter.MetaMode.MIXED);//重写了MetaManager，此处随便设置一个值即可，不设置会报异常
    canalParameter.setIndexMode(CanalParameter.IndexMode.MEMORY_META_FAILBACK);//限定为MEMORY_META_FAILBACK，有需要时再改为可支持其它模式
    canalParameter.setDefaultConnectionTimeoutInSeconds(parameter.getDefaultConnectionTimeoutInSeconds());
    canalParameter.setSendBufferSize(parameter.getSendBufferSize());
    canalParameter.setReceiveBufferSize(parameter.getReceiveBufferSize());
    canalParameter.setFallbackIntervalInSeconds(parameter.getFallbackIntervalInSeconds());
    canalParameter.setBlackFilter(parameter.getBlackFilter());
    canalParameter.setDdlIsolation(true);//必须设置为true，让ddl和dml分属不同的batch
    canalParameter.setMemoryStorageRawEntry(false);//嵌入式模式运行，不需要client和server间的网络通信，所以不需要开启raw模式
    canalParameter.setParallel(parameter.isParallel());//默认false
    canalParameter.setGtidEnable(parameter.isGtidEnable());//默认false
    canalParameter.setFilterTableError(false);
    canalParameter.setSlaveId(-1L);
    return canalParameter;
}
Example #11
Source File: CanalServerWithEmbedded_FileModeTest.java    From canal with Apache License 2.0 4 votes vote down vote up 
protected Canal buildCanal() {
    Canal canal = new Canal();
    canal.setId(1L);
    canal.setName(DESTINATION);
    canal.setDesc("my standalone server test ");

    CanalParameter parameter = new CanalParameter();

    parameter.setMetaMode(MetaMode.LOCAL_FILE);
    parameter.setDataDir("./conf");
    parameter.setMetaFileFlushPeriod(1000);
    parameter.setHaMode(HAMode.HEARTBEAT);
    parameter.setIndexMode(IndexMode.MEMORY_META_FAILBACK);

    parameter.setStorageMode(StorageMode.MEMORY);
    parameter.setMemoryStorageBufferSize(32 * 1024);

    parameter.setSourcingType(SourcingType.MYSQL);
    parameter.setDbAddresses(Arrays.asList(new InetSocketAddress(MYSQL_ADDRESS, 3306),
        new InetSocketAddress(MYSQL_ADDRESS, 3306)));
    parameter.setDbUsername(USERNAME);
    parameter.setDbPassword(PASSWORD);
    parameter.setPositions(Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}",
        "{\"journalName\":\"mysql-bin.000001\",\"position\":332L,\"timestamp\":\"1505998863000\"}"));

    parameter.setSlaveId(1234L);

    parameter.setDefaultConnectionTimeoutInSeconds(30);
    parameter.setConnectionCharset("UTF-8");
    parameter.setConnectionCharsetNumber((byte) 33);
    parameter.setReceiveBufferSize(8 * 1024);
    parameter.setSendBufferSize(8 * 1024);

    parameter.setDetectingEnable(false);
    parameter.setDetectingIntervalInSeconds(10);
    parameter.setDetectingRetryTimes(3);
    parameter.setDetectingSQL(DETECTING_SQL);

    canal.setCanalParameter(parameter);
    return canal;
}