com.alibaba.otter.canal.parse.inbound.TableMeta Java Examples

The following examples show how to use com.alibaba.otter.canal.parse.inbound.TableMeta. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

6 votes

@Test
public void test_any() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_any.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);
    
    List<String> tableNames = Lists.newArrayList();
    for (Schema schema : memoryTableMeta.getRepository().getSchemas()) {
        tableNames.addAll(schema.showTables());
    }

    for (String table : tableNames) {
        TableMeta sourceMeta = memoryTableMeta.find("test", table);
        System.out.println(sourceMeta.toString());
    }
}

Example #2

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

6 votes

@Test
public void test1() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test1.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertNotNull(meta.getFieldMetaByName("customization_id"));

    meta = memoryTableMeta.find("yushitai_test", "_card_record_gho");
    Assert.assertNull(meta);
}

Example #3

Source File: MemoryTableMeta_DDL_Test.java From canal-1.1.3 with Apache License 2.0

6 votes

@Test
public void test1() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test1.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertNotNull(meta.getFieldMetaByName("customization_id"));

    meta = memoryTableMeta.find("yushitai_test", "_card_record_gho");
    Assert.assertNull(meta);
}

Example #4

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #5

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache, EntryPosition position) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache, position);
    } catch (Throwable e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            } else if (StringUtils.contains(message, "errorNumber=1142")
                       && StringUtils.contains(message, "command denied")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #6

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

5 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache);
    } catch (Exception e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #7

Source File: LogEventConvert.java From DBus with Apache License 2.0

5 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache);
    } catch (Exception e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #8

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    lock.readLock().lock();
    try {
        return memoryTableMeta.find(schema, table);
    } finally {
        lock.readLock().unlock();
    }
}

Example #9

Source File: DatabaseTableMeta.java From canal with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    lock.readLock().lock();
    try {
        return memoryTableMeta.find(schema, table);
    } finally {
        lock.readLock().unlock();
    }
}

Example #10

Source File: MemoryTableMeta.java From canal with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    List<String> keys = Arrays.asList(schema, table);
    TableMeta tableMeta = tableMetas.get(keys);
    if (tableMeta == null) {
        synchronized (this) {
            tableMeta = tableMetas.get(keys);
            if (tableMeta == null) {
                Schema schemaRep = repository.findSchema(schema);
                if (schemaRep == null) {
                    return null;
                }
                SchemaObject data = schemaRep.findTable(table);
                if (data == null) {
                    return null;
                }
                SQLStatement statement = data.getStatement();
                if (statement == null) {
                    return null;
                }
                if (statement instanceof SQLCreateTableStatement) {
                    tableMeta = parse((SQLCreateTableStatement) statement);
                }
                if (tableMeta != null) {
                    if (table != null) {
                        tableMeta.setTable(table);
                    }
                    if (schema != null) {
                        tableMeta.setSchema(schema);
                    }

                    tableMetas.put(keys, tableMeta);
                }
            }
        }
    }

    return tableMeta;
}

Example #11

Source File: MemoryTableMeta.java From canal with Apache License 2.0

5 votes

private TableMeta parse(SQLCreateTableStatement statement) {
    int size = statement.getTableElementList().size();
    if (size > 0) {
        TableMeta tableMeta = new TableMeta();
        for (int i = 0; i < size; ++i) {
            SQLTableElement element = statement.getTableElementList().get(i);
            processTableElement(element, tableMeta);
        }
        return tableMeta;
    }

    return null;
}

Example #12

Source File: TableMetaCache.java From canal with Apache License 2.0

5 votes

public static List<FieldMeta> parseTableMeta(String schema, String table, ResultSetPacket packet) {
    if (packet.getFieldValues().size() > 1) {
        String createDDL = packet.getFieldValues().get(1);
        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
        memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
        TableMeta tableMeta = memoryTableMeta.find(schema, table);
        return tableMeta.getFields();
    } else {
        return new ArrayList<FieldMeta>();
    }
}

Example #13

Source File: TableMetaCache.java From canal with Apache License 2.0

5 votes

public TableMeta getTableMeta(String schema, String table, boolean useCache) {
    if (!useCache) {
        tableMetaDB.invalidate(getFullName(schema, table));
    }

    return tableMetaDB.getUnchecked(getFullName(schema, table));
}

Example #14

Source File: MemoryTableMeta_Random_DDL_Test.java From canal with Apache License 2.0

5 votes

private void compareTableMeta(int num, MemoryTableMeta source, MemoryTableMeta target) {
    List<String> tableNames = Lists.newArrayList();
    for (Schema schema : source.getRepository().getSchemas()) {
        tableNames.addAll(schema.showTables());
    }

    for (String table : tableNames) {
        TableMeta sourceMeta = source.find("test", table);
        TableMeta targetMeta = target.find("test", table);
        boolean result = DatabaseTableMeta.compareTableMeta(sourceMeta, targetMeta);
        if (!result) {
            Assert.fail(sourceMeta.toString() + " vs " + targetMeta.toString());
        }
    }
}

Example #15

Source File: MemoryTableMetaTest.java From canal with Apache License 2.0

5 votes

@Test
public void testSimple() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "test");
    System.out.println(meta);
    Assert.assertTrue(meta.getFieldMetaByName("ID").isKey());
}

Example #16

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

5 votes

@Test
public void test2() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test2.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertEquals(meta.getFieldMetaByName("id").isKey(), true);
    Assert.assertEquals(meta.getFieldMetaByName("name").isUnique(), true);
}

Example #17

Source File: MemoryTableMeta_DDL_Test.java From canal with Apache License 2.0

5 votes

@Test
public void test3() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test3.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "quniya4");
    System.out.println(meta);
    Assert.assertTrue(meta.getFields().get(0).getColumnName().equalsIgnoreCase("id"));
}

Example #18

Source File: MemoryTableMeta_DDL_Test.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void test3() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test3.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "quniya4");
    System.out.println(meta);
    Assert.assertTrue(meta.getFields().get(0).getColumnName().equalsIgnoreCase("id"));
}

Example #19

Source File: MemoryTableMeta_DDL_Test.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void test2() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "ddl_test2.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("yushitai_test", "card_record");
    System.out.println(meta);
    Assert.assertEquals(meta.getFieldMetaByName("id").isKey(), true);
    Assert.assertEquals(meta.getFieldMetaByName("name").isUnique(), true);
}

Example #20

Source File: MemoryTableMetaTest.java From canal-1.1.3 with Apache License 2.0

5 votes

@Test
public void testSimple() throws Throwable {
    MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
    String sql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    memoryTableMeta.apply(null, "test", sql, null);

    TableMeta meta = memoryTableMeta.find("test", "test");
    System.out.println(meta);
    Assert.assertTrue(meta.getFieldMetaByName("ID").isKey());
}

Example #21

Source File: LogEventConvert.java From DBus with Apache License 2.0

5 votes

private TableMeta getTableMeta(String dbName, String tbName, boolean useCache) {
    try {
        return tableMetaCache.getTableMeta(dbName, tbName, useCache);
    } catch (Exception e) {
        String message = ExceptionUtils.getRootCauseMessage(e);
        if (filterTableError) {
            if (StringUtils.contains(message, "errorNumber=1146") && StringUtils.contains(message, "doesn't exist")) {
                return null;
            }
        }

        throw new CanalParseException(e);
    }
}

Example #22

Source File: MemoryTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

@Override
public TableMeta find(String schema, String table) {
    List<String> keys = Arrays.asList(schema, table);
    TableMeta tableMeta = tableMetas.get(keys);
    if (tableMeta == null) {
        synchronized (this) {
            tableMeta = tableMetas.get(keys);
            if (tableMeta == null) {
                Schema schemaRep = repository.findSchema(schema);
                if (schemaRep == null) {
                    return null;
                }
                SchemaObject data = schemaRep.findTable(table);
                if (data == null) {
                    return null;
                }
                SQLStatement statement = data.getStatement();
                if (statement == null) {
                    return null;
                }
                if (statement instanceof SQLCreateTableStatement) {
                    tableMeta = parse((SQLCreateTableStatement) statement);
                }
                if (tableMeta != null) {
                    if (table != null) {
                        tableMeta.setTable(table);
                    }
                    if (schema != null) {
                        tableMeta.setSchema(schema);
                    }

                    tableMetas.put(keys, tableMeta);
                }
            }
        }
    }

    return tableMeta;
}

Example #23

Source File: MemoryTableMeta.java From canal-1.1.3 with Apache License 2.0

5 votes

private TableMeta parse(SQLCreateTableStatement statement) {
    int size = statement.getTableElementList().size();
    if (size > 0) {
        TableMeta tableMeta = new TableMeta();
        for (int i = 0; i < size; ++i) {
            SQLTableElement element = statement.getTableElementList().get(i);
            processTableElement(element, tableMeta);
        }
        return tableMeta;
    }

    return null;
}

Example #24

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

5 votes

public static List<FieldMeta> parseTableMeta(String schema, String table, ResultSetPacket packet) {
    if (packet.getFieldValues().size() > 1) {
        String createDDL = packet.getFieldValues().get(1);
        MemoryTableMeta memoryTableMeta = new MemoryTableMeta();
        memoryTableMeta.apply(DatabaseTableMeta.INIT_POSITION, schema, createDDL, null);
        TableMeta tableMeta = memoryTableMeta.find(schema, table);
        return tableMeta.getFields();
    } else {
        return new ArrayList<FieldMeta>();
    }
}

Example #25

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

5 votes

public TableMeta getTableMeta(String schema, String table, boolean useCache) {
    if (!useCache) {
        tableMetaDB.invalidate(getFullName(schema, table));
    }

    return tableMetaDB.getUnchecked(getFullName(schema, table));
}

Example #26

Source File: DatabaseTableMeta.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean compareTableMeta(TableMeta source, TableMeta target) {
    if (!StringUtils.equalsIgnoreCase(source.getSchema(), target.getSchema())) {
        return false;
    }

    if (!StringUtils.equalsIgnoreCase(source.getTable(), target.getTable())) {
        return false;
    }

    List<FieldMeta> sourceFields = source.getFields();
    List<FieldMeta> targetFields = target.getFields();
    if (sourceFields.size() != targetFields.size()) {
        return false;
    }

    for (int i = 0; i < sourceFields.size(); i++) {
        FieldMeta sourceField = sourceFields.get(i);
        FieldMeta targetField = targetFields.get(i);
        if (!StringUtils.equalsIgnoreCase(sourceField.getColumnName(), targetField.getColumnName())) {
            return false;
        }

        // if (!StringUtils.equalsIgnoreCase(sourceField.getColumnType(),
        // targetField.getColumnType())) {
        // return false;
        // }

        // https://github.com/alibaba/canal/issues/1100
        // 支持一下 int vs int(10)
        if ((sourceField.isUnsigned() && !targetField.isUnsigned())
            || (!sourceField.isUnsigned() && targetField.isUnsigned())) {
            return false;
        }

        String sign = sourceField.isUnsigned() ? "unsigned" : "signed";
        String sourceColumnType = StringUtils.removeEndIgnoreCase(sourceField.getColumnType(), sign).trim();
        String targetColumnType = StringUtils.removeEndIgnoreCase(targetField.getColumnType(), sign).trim();

        boolean columnTypeCompare = false;
        columnTypeCompare |= StringUtils.containsIgnoreCase(sourceColumnType, targetColumnType);
        columnTypeCompare |= StringUtils.containsIgnoreCase(targetColumnType, sourceColumnType);
        if (!columnTypeCompare) {
            return false;
        }

        // if (!StringUtils.equalsIgnoreCase(sourceField.getDefaultValue(),
        // targetField.getDefaultValue())) {
        // return false;
        // }

        if (sourceField.isNullable() != targetField.isNullable()) {
            return false;
        }

        // mysql会有一种处理,针对show create只有uk没有pk时，会在desc默认将uk当做pk
        boolean isSourcePkOrUk = sourceField.isKey() || sourceField.isUnique();
        boolean isTargetPkOrUk = targetField.isKey() || targetField.isUnique();
        if (isSourcePkOrUk != isTargetPkOrUk) {
            return false;
        }
    }

    return true;
}

Example #27

Source File: TableMetaCache.java From canal-1.1.3 with Apache License 2.0

4 votes

public TableMeta getTableMeta(String schema, String table) {
    return getTableMeta(schema, table, true);
}

Example #28

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

4 votes

public void setTable(TableMeta table) {
    this.table = table;
}

Example #29

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

4 votes

public TableMeta getTable() {
    return table;
}

Example #30

Source File: MysqlMultiStageCoprocessor.java From canal with Apache License 2.0

4 votes

public void onEvent(MessageEvent event, long sequence, boolean endOfBatch) throws Exception {
    try {
        LogEvent logEvent = event.getEvent();
        if (logEvent == null) {
            LogBuffer buffer = event.getBuffer();
            logEvent = decoder.decode(buffer, context);
            event.setEvent(logEvent);
        }

        int eventType = logEvent.getHeader().getType();
        TableMeta tableMeta = null;
        boolean needDmlParse = false;
        switch (eventType) {
            case LogEvent.WRITE_ROWS_EVENT_V1:
            case LogEvent.WRITE_ROWS_EVENT:
                tableMeta = logEventConvert.parseRowsEventForTableMeta((WriteRowsLogEvent) logEvent);
                needDmlParse = true;
                break;
            case LogEvent.UPDATE_ROWS_EVENT_V1:
            case LogEvent.PARTIAL_UPDATE_ROWS_EVENT:
            case LogEvent.UPDATE_ROWS_EVENT:
                tableMeta = logEventConvert.parseRowsEventForTableMeta((UpdateRowsLogEvent) logEvent);
                needDmlParse = true;
                break;
            case LogEvent.DELETE_ROWS_EVENT_V1:
            case LogEvent.DELETE_ROWS_EVENT:
                tableMeta = logEventConvert.parseRowsEventForTableMeta((DeleteRowsLogEvent) logEvent);
                needDmlParse = true;
                break;
            case LogEvent.ROWS_QUERY_LOG_EVENT:
                needDmlParse = true;
                break;
            default:
                CanalEntry.Entry entry = logEventConvert.parse(event.getEvent(), false);
                event.setEntry(entry);
        }

        // 记录一下DML的表结构
        event.setNeedDmlParse(needDmlParse);
        event.setTable(tableMeta);
    } catch (Throwable e) {
        exception = new CanalParseException(e);
        throw exception;
    }
}