com.alibaba.otter.canal.protocol.CanalEntry.Column Java Examples

The following examples show how to use com.alibaba.otter.canal.protocol.CanalEntry.Column. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #2

Source File: LogEventConvert.java From canal with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #3

Source File: BaseCanalClientTest.java From canal with Apache License 2.0

6 votes

protected void printColumn(List<Column> columns) {
    for (Column column : columns) {
        StringBuilder builder = new StringBuilder();
        try {
            if (StringUtils.containsIgnoreCase(column.getMysqlType(), "BLOB")
                || StringUtils.containsIgnoreCase(column.getMysqlType(), "BINARY")) {
                // get value bytes
                builder.append(column.getName() + " : "
                               + new String(column.getValue().getBytes("ISO-8859-1"), "UTF-8"));
            } else {
                builder.append(column.getName() + " : " + column.getValue());
            }
        } catch (UnsupportedEncodingException e) {
        }
        builder.append("    type=" + column.getMysqlType());
        if (column.getUpdated()) {
            builder.append("    update=" + column.getUpdated());
        }
        builder.append(SEP);
        logger.info(builder.toString());
    }
}

Example #4

Source File: LogEventConvert.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #5

Source File: LogEventConvert_old.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// �Ƚ�before / after��column index
            if (column.getIsNull() && newValue == null) {
                // ���ȫ��null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // �����Ϊnull���������
                return false;
            }
        }
    }

    // ����nolob/minialģʽ��,�����Ҳ���before��¼,��Ϊ���б仯
    return true;
}

Example #6

Source File: LogEventConvert.java From DBus with Apache License 2.0

6 votes

private boolean isUpdate(List<Column> bfColumns, String newValue, int index) {
    if (bfColumns == null) {
        throw new CanalParseException("ERROR ## the bfColumns is null");
    }

    if (index < 0) {
        return false;
    }

    for (Column column : bfColumns) {
        if (column.getIndex() == index) {// 比较before / after的column index
            if (column.getIsNull() && newValue == null) {
                // 如果全是null
                return false;
            } else if (newValue != null && (!column.getIsNull() && column.getValue().equals(newValue))) {
                // fixed issue #135, old column is Null
                // 如果不为null，并且相等
                return false;
            }
        }
    }

    // 比如nolob/minial模式下,可能找不到before记录,认为是有变化
    return true;
}

Example #7

Source File: MysqlWrapperDefaultHandler.java From DBus with Apache License 2.0

6 votes

/**
 * 创建DBusMessageBuilder对象,同时生成ums schema
 */
private DbusMessageBuilder createBuilderWithSchema(MetaVersion version, MessageEntry msgEntry) {
    DbusMessageBuilder builder = new DbusMessageBuilder();
    String namespace;
    if (outputTablePartition()) {
        namespace = builder.buildNameSpace(Utils.getDataSourceNamespace(),
                msgEntry.getEntryHeader().getSchemaName(), msgEntry.getEntryHeader().getTableName(),
                version.getVersion(), msgEntry.getEntryHeader().getPartitionTableName());
    } else {
        namespace = builder.buildNameSpace(Utils.getDataSourceNamespace(),
                msgEntry.getEntryHeader().getSchemaName(), msgEntry.getEntryHeader().getTableName(),
                version.getVersion());
    }
    builder.build(DbusMessage.ProtocolType.DATA_INCREMENT_DATA, namespace, table.getBatchId());
    EventType eventType = msgEntry.getEntryHeader().getOperType();
    RowData rowData = msgEntry.getMsgColumn().getRowDataLst().get(0);
    List<Column> columns = Support.getFinalColumns(eventType, rowData);
    for (Column column : columns) {
        String colType = Support.getColumnType(column);
        // 这里避免，源端表中包含ums_id_/ums_ts_等字段
        if (Support.isSupported(colType) && !builder.getMessage().containsFiled(column.getName())) {
            builder.appendSchema(column.getName(), DataType.convertMysqlDataType(colType), true);
        }
    }
    return builder;
}

Example #8

Source File: MysqlWrapperDefaultHandler.java From DBus with Apache License 2.0

6 votes

private int addPayloadColumns(List<Object> payloads, List<Column> columns,
                              PairWrapper<String, Object> wrapper, DbusMessageBuilder builder) throws Exception {
    int payloadSize = 0;
    //Map<String, Column> map = new HashMap<>();
    //for (Column column : columns) {
    //    map.put(column.getName(), column);
    //}
    List<DbusMessage.Field> fields = builder.getMessage().getSchema().getFields();

    for (int i = builder.getUmsFixedFields(); i < fields.size(); i++) {
        DbusMessage.Field field = fields.get(i);
        // 这里避免，源端表中包含ums_id_/ums_ts_等字段
        //Column column = map.get(field.getName());
        //if (column != null && Support.isSupported(column)) {
            Pair<String, Object> pair = wrapper.getPair(field.getName());
            Object value = pair.getValue();
            payloads.add(value);
            if (value != null) {
                payloadSize += value.toString().getBytes("utf-8").length;
            }
        //}
    }
    return payloadSize;
}

Example #9

Source File: Support.java From DBus with Apache License 2.0

6 votes

public static long[] getColumnLengthAndPrecision(Column column) {
    long[] ret = new long[2];
    String data = StringUtils.substringBetween(column.getMysqlType(), "(", ")");
    String length = StringUtils.substringBefore(data, ",");
    String precision = StringUtils.substringAfter(data, ",");
    String type = StringUtils.substringBefore(column.getMysqlType(), "(").toUpperCase();
    if ("SET".equals(type) || "ENUM".equals(type)) {
        ret[0] = 0;
        ret[1] = 0;
    } else {
        if (StringUtils.isEmpty(length)) {
            ret[0] = 0;
        } else {
            ret[0] = Long.parseLong(length);
        }
        if (StringUtils.isEmpty(precision)) {
            ret[1] = 0;
        } else {
            ret[1] = Long.parseLong(precision);
        }
    }
    return ret;
}

Example #10

Source File: Convertor.java From DBus with Apache License 2.0

6 votes

public static <T extends Object> PairWrapper<String, Object> convertProtobufRecord(EntryHeader header, RowData rowData) {
    PairWrapper<String, Object> wrapper = new PairWrapper<>();
    List<Column> Columns = null;
    if (header.isInsert() || header.isUpdate()) {
        Columns = rowData.getAfterColumnsList();
    } else if (header.isDelete()) {
        Columns = rowData.getBeforeColumnsList();
    }
    wrapper.addProperties(Constants.MessageBodyKey.POS, header.getPos());
    wrapper.addProperties(Constants.MessageBodyKey.OP_TS, header.getTsTime());

    Map<String, Object> map = convert2map(Columns);
    for (Map.Entry<String, Object> entry : map.entrySet()) {
        wrapper.addPair(new Pair<>(entry.getKey(), CharSequence.class.isInstance(entry.getValue()) ? entry.getValue().toString() : entry.getValue()));
    }

    return wrapper;
}

Example #11

Source File: BaseCanalClientTest.java From canal-1.1.3 with Apache License 2.0

6 votes

protected void printColumn(List<Column> columns) {
    for (Column column : columns) {
        StringBuilder builder = new StringBuilder();
        try {
            if (StringUtils.containsIgnoreCase(column.getMysqlType(), "BLOB")
                || StringUtils.containsIgnoreCase(column.getMysqlType(), "BINARY")) {
                // get value bytes
                builder.append(column.getName() + " : "
                               + new String(column.getValue().getBytes("ISO-8859-1"), "UTF-8"));
            } else {
                builder.append(column.getName() + " : " + column.getValue());
            }
        } catch (UnsupportedEncodingException e) {
        }
        builder.append("    type=" + column.getMysqlType());
        if (column.getUpdated()) {
            builder.append("    update=" + column.getUpdated());
        }
        builder.append(SEP);
        logger.info(builder.toString());
    }
}

Example #12

Source File: Support.java From DBus with Apache License 2.0

5 votes

public static List<Column> getFinalColumns(EventType type, RowData rowData) {
    List<Column> Columns = null;
    if (type == EventType.INSERT || type == EventType.UPDATE) {
        Columns = rowData.getAfterColumnsList();
    } else if (type == EventType.DELETE) {
        Columns = rowData.getBeforeColumnsList();
    }
    return Columns;
}

Example #13

Source File: Support.java From DBus with Apache License 2.0

5 votes

public static String getColumnType(Column column) {
    String type = column.getMysqlType().toLowerCase();
    if (type.startsWith("int(") && type.endsWith("unsigned")) {
        return "int unsigned";
    }
    return StringUtils.substringBefore(column.getMysqlType(), "(");
}

Example #14

Source File: Convertor.java From DBus with Apache License 2.0

5 votes

private static Map<String, Object> convert2map(List<Column> cols) {
    Map<String, Object> map = Maps.newHashMap();
    if (cols != null) {
        for (Column col : cols) {
            String colName = col.getName();
            if (col.getIsNull()) {
                map.put(colName, null);
            } else {
                map.put(colName, col.getValue());
            }
        }
    }
    return map;
}

Example #15

Source File: Convertor.java From DBus with Apache License 2.0

5 votes

public static <T extends Object> PairWrapper<String, Object> convertProtobufRecordBeforeUpdate(EntryHeader header, RowData rowData) {
    PairWrapper<String, Object> wrapper = new PairWrapper<>();
    List<Column> Columns = rowData.getBeforeColumnsList();
    Map<String, Object> map = convert2map(Columns);
    for (Map.Entry<String, Object> entry : map.entrySet()) {
        wrapper.addPair(new Pair<>(entry.getKey(), CharSequence.class.isInstance(entry.getValue()) ? entry.getValue().toString() : entry.getValue()));
    }

    return wrapper;
}

Example #16

Source File: CanalReaderMessageParser.java From DataLink with Apache License 2.0

5 votes

/**
 * Transfer Canal-Column to Datalink`s Event-Column
 */
private EventColumn copyEventColumn(Column column) {
    EventColumn eventColumn = new EventColumn();
    eventColumn.setIndex(column.getIndex());
    eventColumn.setKey(column.getIsKey());
    eventColumn.setNull(column.getIsNull());
    eventColumn.setColumnName(column.getName());
    eventColumn.setColumnValue(column.getValue());
    eventColumn.setUpdate(column.getUpdated());
    eventColumn.setColumnType(column.getSqlType());

    return eventColumn;
}

Example #17

Source File: UpdateCanalListener.java From canal_mysql_elasticsearch_sync with Apache License 2.0

5 votes

@Override
protected void doSync(String database, String table, String index, String type, RowData rowData) {
    List<Column> columns = rowData.getAfterColumnsList();
    String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(database + "." + table)).orElse("id");
    Column idColumn = columns.stream().filter(column -> column.getIsKey() && primaryKey.equals(column.getName())).findFirst().orElse(null);
    if (idColumn == null || StringUtils.isBlank(idColumn.getValue())) {
        logger.warn("update_column_find_null_warn update从column中找不到主键,database=" + database + ",table=" + table);
        return;
    }
    logger.debug("update_column_id_info update主键id,database=" + database + ",table=" + table + ",id=" + idColumn.getValue());
    Map<String, Object> dataMap = parseColumnsToMap(columns);
    elasticsearchService.update(index, type, idColumn.getValue(), dataMap);
    logger.debug("update_es_info 同步es插入操作成功！database=" + database + ",table=" + table + ",data=" + dataMap);
}

Example #18

Source File: InsertCanalListener.java From canal_mysql_elasticsearch_sync with Apache License 2.0

5 votes

@Override
protected void doSync(String database, String table, String index, String type, RowData rowData) {
    List<Column> columns = rowData.getAfterColumnsList();
    String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(database + "." + table)).orElse("id");
    Column idColumn = columns.stream().filter(column -> column.getIsKey() && primaryKey.equals(column.getName())).findFirst().orElse(null);
    if (idColumn == null || StringUtils.isBlank(idColumn.getValue())) {
        logger.warn("insert_column_find_null_warn insert从column中找不到主键,database=" + database + ",table=" + table);
        return;
    }
    logger.debug("insert_column_id_info insert主键id,database=" + database + ",table=" + table + ",id=" + idColumn.getValue());
    Map<String, Object> dataMap = parseColumnsToMap(columns);
    elasticsearchService.insertById(index, type, idColumn.getValue(), dataMap);
    logger.debug("insert_es_info 同步es插入操作成功！database=" + database + ",table=" + table + ",data=" + JsonUtil.toJson(dataMap));
}

Example #19

Source File: DeleteCanalListener.java From canal_mysql_elasticsearch_sync with Apache License 2.0

5 votes

@Override
protected void doSync(String database, String table, String index, String type, RowData rowData) {
    List<Column> columns = rowData.getBeforeColumnsList();
    String primaryKey = Optional.ofNullable(mappingService.getTablePrimaryKeyMap().get(database + "." + table)).orElse("id");
    Column idColumn = columns.stream().filter(column -> column.getIsKey() && primaryKey.equals(column.getName())).findFirst().orElse(null);
    if (idColumn == null || StringUtils.isBlank(idColumn.getValue())) {
        logger.warn("insert_column_find_null_warn insert从column中找不到主键,database=" + database + ",table=" + table);
        return;
    }
    logger.debug("insert_column_id_info insert主键id,database=" + database + ",table=" + table + ",id=" + idColumn.getValue());
    elasticsearchService.deleteById(index, type, idColumn.getValue());
    logger.debug("insert_es_info 同步es插入操作成功！database=" + database + ",table=" + table + ",id=" + idColumn.getValue());
}

Example #20

Source File: AbstractCanalListener.java From canal_mysql_elasticsearch_sync with Apache License 2.0

5 votes

Map<String, Object> parseColumnsToMap(List<Column> columns) {
    Map<String, Object> jsonMap = new HashMap<>();
    columns.forEach(column -> {
        if (column == null) {
            return;
        }
        jsonMap.put(column.getName(), column.getIsNull() ? null : mappingService.getElasticsearchTypeObject(column.getMysqlType(), column.getValue()));
    });
    return jsonMap;
}

Example #21

Source File: LocalBinlogDumpTest.java From canal with Apache License 2.0

4 votes

private void print(List<Column> columns) {
    for (Column column : columns) {
        System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
    }
}

Example #22

Source File: MysqlDumpTest.java From canal with Apache License 2.0

4 votes

private void print(List<Column> columns) {
    for (Column column : columns) {
        System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
    }
}

Example #23

Source File: RdsLocalBinlogDumpTest.java From canal with Apache License 2.0

4 votes

private void print(List<Column> columns) {
    for (Column column : columns) {
        System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
    }
}

Example #24

Source File: RdsLocalBinlogDumpTest.java From canal-1.1.3 with Apache License 2.0

4 votes

private void print(List<Column> columns) {
    for (Column column : columns) {
        System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
    }
}

Example #25

Source File: MysqlDumpTest.java From canal-1.1.3 with Apache License 2.0

4 votes

private void print(List<Column> columns) {
    for (Column column : columns) {
        System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
    }
}

Example #26

Source File: LocalBinlogDumpTest.java From canal-1.1.3 with Apache License 2.0

4 votes

private void print(List<Column> columns) {
    for (Column column : columns) {
        System.out.println(column.getName() + " : " + column.getValue() + "    update=" + column.getUpdated());
    }
}

Example #27

Source File: Support.java From DBus with Apache License 2.0

4 votes

public static boolean isSupported(Column column) {
    String type = StringUtils.substringBefore(column.getMysqlType(), "(");
    return SupportedMysqlDataType.isSupported(type);
}