com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter Java Examples

The following examples show how to use com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: AbstractCanalInstance.java From canal-1.1.3 with Apache License 2.0

6 votes

@Override
public boolean subscribeChange(ClientIdentity identity) {
    if (StringUtils.isNotEmpty(identity.getFilter())) {
        logger.info("subscribe filter change to " + identity.getFilter());
        AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());

        boolean isGroup = (eventParser instanceof GroupEventParser);
        if (isGroup) {
            // 处理group的模式
            List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
                ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
            }
        } else {
            ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
        }

    }

    // filter的处理规则
    // a. parser处理数据过滤处理
    // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
    // 后续内存版的一对多分发，可以考虑
    return true;
}

Example #2

Source File: MQMessageUtils.java From canal-1.1.3 with Apache License 2.0

6 votes

public List<DynamicTopicData> apply(String pkHashConfigs) {
    List<DynamicTopicData> datas = Lists.newArrayList();
    String[] dynamicTopicArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
        ",",
        ";"),
        ";");
    // schema.table
    for (String dynamicTopic : dynamicTopicArray) {
        DynamicTopicData data = new DynamicTopicData();

        if (!isWildCard(dynamicTopic)) {
            data.simpleName = dynamicTopic;
        } else {
            if (dynamicTopic.contains("\\.")) {
                data.tableRegexFilter = new AviaterRegexFilter(dynamicTopic);
            } else {
                data.schemaRegexFilter = new AviaterRegexFilter(dynamicTopic);
            }
        }
        datas.add(data);
    }

    return datas;
}

Example #3

Source File: AviaterFilterTest.java From canal with Apache License 2.0

6 votes

@Test
public void testDisordered() {
    AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");

    boolean result = filter.filter("u.abc");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.x");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.xyz1");
    Assert.assertEquals(false, result);

    result = filter.filter("abc.xyz");
    Assert.assertEquals(true, result);

    result = filter.filter("uv.xyz");
    Assert.assertEquals(true, result);

}

Example #4

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

6 votes

protected BinlogParser buildParser() {
    LogEventConvert convert = new LogEventConvert();
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        convert.setNameFilter((AviaterRegexFilter) eventFilter);
    }

    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
    }

    convert.setCharset(connectionCharset);
    convert.setFilterQueryDcl(filterQueryDcl);
    convert.setFilterQueryDml(filterQueryDml);
    convert.setFilterQueryDdl(filterQueryDdl);
    convert.setFilterRows(filterRows);
    convert.setFilterTableError(filterTableError);
    convert.setUseDruidDdlFilter(useDruidDdlFilter);
    return convert;
}

Example #5

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

6 votes

protected BinlogParser buildParser() {
    LogEventConvert convert = new LogEventConvert();
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        convert.setNameFilter((AviaterRegexFilter) eventFilter);
    }

    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
    }
    
    convert.setFieldFilterMap(getFieldFilterMap());
    convert.setFieldBlackFilterMap(getFieldBlackFilterMap());

    convert.setCharset(connectionCharset);
    convert.setFilterQueryDcl(filterQueryDcl);
    convert.setFilterQueryDml(filterQueryDml);
    convert.setFilterQueryDdl(filterQueryDdl);
    convert.setFilterRows(filterRows);
    convert.setFilterTableError(filterTableError);
    convert.setUseDruidDdlFilter(useDruidDdlFilter);
    return convert;
}

Example #6

Source File: AviaterFilterTest.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void testDisordered() {
    AviaterRegexFilter filter = new AviaterRegexFilter("u\\..*,uvw\\..*,uv\\..*,a\\.x,a\\.xyz,a\\.xy,abc\\.x,abc\\.xyz,abc\\.xy,ab\\.x,ab\\.xyz,ab\\.xy");

    boolean result = filter.filter("u.abc");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.x");
    Assert.assertEquals(true, result);

    result = filter.filter("ab.xyz1");
    Assert.assertEquals(false, result);

    result = filter.filter("abc.xyz");
    Assert.assertEquals(true, result);

    result = filter.filter("uv.xyz");
    Assert.assertEquals(true, result);

}

Example #7

Source File: MQMessageUtils.java From canal-1.1.3 with Apache License 2.0

5 votes

public List<PartitionData> apply(String pkHashConfigs) {
    List<PartitionData> datas = Lists.newArrayList();

    String[] pkHashConfigArray = StringUtils.split(StringUtils.replace(pkHashConfigs,
        ",",
        ";"),
        ";");
    // schema.table:id^name
    for (String pkHashConfig : pkHashConfigArray) {
        PartitionData data = new PartitionData();
        int i = pkHashConfig.lastIndexOf(":");
        if (i > 0) {
            String pkStr = pkHashConfig.substring(i + 1);
            if (pkStr.equalsIgnoreCase("$pk$")) {
                data.hashMode.autoPkHash = true;
            } else {
                data.hashMode.pkNames = Lists.newArrayList(StringUtils.split(pkStr,
                    '^'));
            }

            pkHashConfig = pkHashConfig.substring(0,
                i);
        } else {
            data.hashMode.tableHash = true;
        }

        if (!isWildCard(pkHashConfig)) {
            data.simpleName = pkHashConfig;
        } else {
            data.regexFilter = new AviaterRegexFilter(pkHashConfig);
        }
        datas.add(data);
    }

    return datas;
}

Example #8

Source File: MutliAviaterFilterTest.java From canal with Apache License 2.0

5 votes

private void doRegexTest() {
    AviaterRegexFilter filter3 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|"
                                                        + RandomStringUtils.randomAlphabetic(200));
    boolean result = filter3.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
    result = filter3.filter("otter2.otter_stability1");
    Assert.assertEquals(true, result);
}

Example #9

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

5 votes

public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
    super.setEventBlackFilter(eventBlackFilter);

    // 触发一下filter变更
    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        }
    }
}

Example #10

Source File: AbstractMysqlEventParser.java From canal with Apache License 2.0

5 votes

public void setEventFilter(CanalEventFilter eventFilter) {
    super.setEventFilter(eventFilter);

    // 触发一下filter变更
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        }
    }
}

Example #11

Source File: AbstractCanalInstance.java From canal with Apache License 2.0

5 votes

@Override
public boolean subscribeChange(ClientIdentity identity) {
    if (StringUtils.isNotEmpty(identity.getFilter())) {
        logger.info("subscribe filter change to " + identity.getFilter());
        AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());

        boolean isGroup = (eventParser instanceof GroupEventParser);
        if (isGroup) {
            // 处理group的模式
            List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
            for (CanalEventParser singleEventParser : eventParsers) {// 需要遍历启动
                if(singleEventParser instanceof AbstractEventParser) {
                    ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
                }
            }
        } else {
            if(eventParser instanceof AbstractEventParser) {
                ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
            }
        }

    }

    // filter的处理规则
    // a. parser处理数据过滤处理
    // b. sink处理数据的路由&分发,一份parse数据经过sink后可以分发为多份，每份的数据可以根据自己的过滤规则不同而有不同的数据
    // 后续内存版的一对多分发，可以考虑
    return true;
}

Example #12

Source File: Binlog.java From jlogstash-input-plugin with Apache License 2.0

5 votes

@Override
public void prepare() {
    logger.info("binlog prepare started..");

    parseCategories();

    controller = new MysqlEventParser();
    controller.setConnectionCharset(Charset.forName("UTF-8"));
    controller.setSlaveId(slaveId);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress(host, port), username, password));
    controller.setEnableTsdb(true);
    controller.setDestination("example");
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(2);
    controller.setIsGTIDMode(false);
    controller.setEventSink(new BinlogEventSink(this));

    controller.setLogPositionManager(new BinlogPositionManager(this));

    EntryPosition startPosition = findStartPosition();
    if (startPosition != null) {
       controller.setMasterPosition(startPosition);
    }


    if (filter != null) {
        controller.setEventFilter(new AviaterRegexFilter(filter));
    }

    logger.info("binlog prepare ended..");
}

Example #13

Source File: MutliAviaterFilterTest.java From canal-1.1.3 with Apache License 2.0

5 votes

private void doRegexTest() {
    AviaterRegexFilter filter3 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|"
                                                        + RandomStringUtils.randomAlphabetic(200));
    boolean result = filter3.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
    result = filter3.filter("otter2.otter_stability1");
    Assert.assertEquals(true, result);
}

Example #14

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

public void setEventBlackFilter(CanalEventFilter eventBlackFilter) {
    super.setEventBlackFilter(eventBlackFilter);

    // 触发一下filter变更
    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
        }
    }
}

Example #15

Source File: AbstractMysqlEventParser.java From canal-1.1.3 with Apache License 2.0

5 votes

public void setEventFilter(CanalEventFilter eventFilter) {
    super.setEventFilter(eventFilter);

    // 触发一下filter变更
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        if (binlogParser instanceof LogEventConvert) {
            ((LogEventConvert) binlogParser).setNameFilter((AviaterRegexFilter) eventFilter);
        }

        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
        }
    }
}

Example #16

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
}

Example #17

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
}

Example #18

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
}

Example #19

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
}

Example #20

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
}

Example #21

Source File: LogEventConvert.java From DBus with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
}

Example #22

Source File: LogEventConvert.java From canal with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
    logger.warn("--> init table filter : " + nameFilter.toString());
}

Example #23

Source File: LogEventConvert.java From canal with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
    logger.warn("--> init table black filter : " + nameBlackFilter.toString());
}

Example #24

Source File: AviaterFilterTest.java From canal-1.1.3 with Apache License 2.0

4 votes

@Test
public void test_regex() {
    AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
    boolean result = filter.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("");
    Assert.assertEquals(true, result);

    result = filter.filter("s12.t1");
    Assert.assertEquals(false, result);

    result = filter.filter("s2.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("s3.t2");
    Assert.assertEquals(false, result);

    result = filter.filter("S1.S2");
    Assert.assertEquals(true, result);

    result = filter.filter("S2.S1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");

    result = filter2.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter2.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter2.filter("s2.t1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");

    result = filter3.filter("fooooot");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter4 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
    result = filter4.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
}

Example #25

Source File: AviaterFilterTest.java From canal with Apache License 2.0

4 votes

@Test
public void test_regex() {
    AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
    boolean result = filter.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("");
    Assert.assertEquals(true, result);

    result = filter.filter("s12.t1");
    Assert.assertEquals(false, result);

    result = filter.filter("s2.t2");
    Assert.assertEquals(true, result);

    result = filter.filter("s3.t2");
    Assert.assertEquals(false, result);

    result = filter.filter("S1.S2");
    Assert.assertEquals(true, result);

    result = filter.filter("S2.S1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");

    result = filter2.filter("s1.t1");
    Assert.assertEquals(true, result);

    result = filter2.filter("s1.t2");
    Assert.assertEquals(true, result);

    result = filter2.filter("s2.t1");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");

    result = filter3.filter("fooooot");
    Assert.assertEquals(true, result);

    AviaterRegexFilter filter4 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
    result = filter4.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
}

Example #26

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

4 votes

public void setNameBlackFilter(AviaterRegexFilter nameBlackFilter) {
    this.nameBlackFilter = nameBlackFilter;
    logger.warn("--> init table black filter : " + nameBlackFilter.toString());
}

Example #27

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

4 votes

public void setNameFilter(AviaterRegexFilter nameFilter) {
    this.nameFilter = nameFilter;
    logger.warn("--> init table filter : " + nameFilter.toString());
}