Java Code Examples for com.alibaba.otter.canal.protocol.CanalEntry.EventType#INSERT

The following examples show how to use com.alibaba.otter.canal.protocol.CanalEntry.EventType#INSERT . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: Support.java From DBus with Apache License 2.0

5 votes

public static List<Column> getFinalColumns(EventType type, RowData rowData) {
    List<Column> Columns = null;
    if (type == EventType.INSERT || type == EventType.UPDATE) {
        Columns = rowData.getAfterColumnsList();
    } else if (type == EventType.DELETE) {
        Columns = rowData.getBeforeColumnsList();
    }
    return Columns;
}

Example 2

Source File: LogEventConvert.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean processFilter(String queryString, DdlResult result) {
    String schemaName = result.getSchemaName();
    String tableName = result.getTableName();
    // fixed issue https://github.com/alibaba/canal/issues/58
    // 更新下table meta cache
    if (tableMetaCache != null
        && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
            String schemaName0 = renameResult.getSchemaName();
            String tableName0 = renameResult.getTableName();
            if (StringUtils.isNotEmpty(tableName0)) {
                // 如果解析到了正确的表信息，则根据全名进行清除
                tableMetaCache.clearTableMeta(schemaName0, tableName0);
            } else {
                // 如果无法解析正确的表信息，则根据schema进行清除
                tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
            }
        }
    }

    // fixed issue https://github.com/alibaba/canal/issues/58
    if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE
        || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE
        || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX
        || result.getType() == EventType.DINDEX) { // 针对DDL类型

        if (filterQueryDdl) {
            return true;
        }

        if (StringUtils.isEmpty(tableName)
            || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
            // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
            throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: "
                                          + queryString + " , and DdlResult: " + result.toString());
            // return null;
        } else {
            // check name filter
            String name = schemaName + "." + tableName;
            if (nameFilter != null && !nameFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameFilter != null
                        && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }

            if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameBlackFilter != null
                        && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }
        }
    } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE
               || result.getType() == EventType.DELETE) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        if (filterQueryDml) {
            return true;
        }
    } else if (filterQueryDcl) {
        return true;
    }

    return false;
}

Example 3

Source File: EventTransactionBuffer.java From canal-1.1.3 with Apache License 2.0

4 votes

private boolean isDml(EventType eventType) {
    return eventType == EventType.INSERT || eventType == EventType.UPDATE || eventType == EventType.DELETE;
}

Example 4

Source File: EntryHeader.java From DBus with Apache License 2.0

4 votes

public boolean isInsert() {
    return header.getEventType() == EventType.INSERT;
}

Example 5

Source File: MysqlInitialLoadProcessor.java From DBus with Apache License 2.0

4 votes

/**
 * 处理mysql 拉全量请求
 */
@Override
public void process(final DBusConsumerRecord<String, byte[]> consumerRecord, Object... args) {
    try {
        List<MessageEntry> msgEntryLst = parser.getEntry(consumerRecord.value());
        if (msgEntryLst.isEmpty())
            return;
        EntryHeader entryHeader = msgEntryLst.get(0).getEntryHeader();
        EventType operType = entryHeader.getOperType();

        //TODO 暂时放弃拉全量表的update/delete等消息
        if (operType != EventType.INSERT) {
            listener.reduceFlowSize(consumerRecord.serializedValueSize());
            consumerListener.syncOffset(consumerRecord);
            return;
        }

        // 判断是否处理过该消息
        String msgPos = entryHeader.getPos();
        Object processed = cache.getIfPresent(msgPos);
        if (processed != null) {
            logger.info("Data have bean processed, the data position is [{}]", msgPos);
            listener.reduceFlowSize(consumerRecord.serializedValueSize());
            consumerListener.syncOffset(consumerRecord);
            return;
        }

        logger.info("Received FULL DATA PULL REQUEST message");
        ControlMessage message = Convertor.mysqlFullPullMessage(msgEntryLst.get(0), listener.getListenerId(), consumerRecord);
        String schemaName = getStringValue("SCHEMA_NAME", message);
        String tableName = getStringValue("TABLE_NAME", message);
        DataTable table = ThreadLocalCache.get(Constants.CacheNames.DATA_TABLES, Utils.buildDataTableCacheKey(schemaName, tableName));
        if (table == null) {
            logger.warn("Table {}.{} is not supported,please configure it in dbus database.", schemaName, tableName);
            return;
        }

        for (String controlTopic : controlTopics) {
            String json = message.toJSONString();
            ProducerRecord<String, byte[]> producerRecord = new ProducerRecord<>(controlTopic, message.getType(), json.getBytes());

            Future<RecordMetadata> future = listener.sendRecord(producerRecord);
            future.get();

            logger.info("write initial load request message to kafka: {}", json);
        }

        // 暂停接收 topic 的数据
        TopicPartition tp = new TopicPartition(consumerRecord.topic(), consumerRecord.partition());
        consumerListener.pauseTopic(tp, consumerRecord.offset(), message);

        // emit FULL_DATA_PULL_REQ 通知给bolt
        EmitData emitData = new EmitData();
        emitData.add(EmitData.DB_SCHEMA, schemaName);
        emitData.add(EmitData.DATA_TABLE, tableName);
        listener.emitData(emitData, Command.FULL_DATA_PULL_REQ, consumerRecord);

        consumerListener.syncOffset(consumerRecord);
        cache.put(msgPos, msgEntryLst.get(0).toString());
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 6

Source File: LogEventConvert.java From canal with Apache License 2.0

4 votes

private boolean processFilter(String queryString, DdlResult result) {
    String schemaName = result.getSchemaName();
    String tableName = result.getTableName();
    // fixed issue https://github.com/alibaba/canal/issues/58
    // 更新下table meta cache
    if (tableMetaCache != null
        && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
            String schemaName0 = renameResult.getSchemaName();
            String tableName0 = renameResult.getTableName();
            if (StringUtils.isNotEmpty(tableName0)) {
                // 如果解析到了正确的表信息，则根据全名进行清除
                tableMetaCache.clearTableMeta(schemaName0, tableName0);
            } else {
                // 如果无法解析正确的表信息，则根据schema进行清除
                tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
            }
        }
    }

    // fixed issue https://github.com/alibaba/canal/issues/58
    if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE
        || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE
        || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX
        || result.getType() == EventType.DINDEX) { // 针对DDL类型

        if (filterQueryDdl) {
            return true;
        }

        if (StringUtils.isEmpty(tableName)
            || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
            // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
            throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: "
                                          + queryString + " , and DdlResult: " + result.toString());
            // return null;
        } else {
            // check name filter
            String name = schemaName + "." + tableName;
            if (nameFilter != null && !nameFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameFilter != null
                        && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }

            if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameBlackFilter != null
                        && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }
        }
    } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE
               || result.getType() == EventType.DELETE) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        if (filterQueryDml) {
            return true;
        }
    } else if (filterQueryDcl) {
        return true;
    }

    return false;
}

Example 7

Source File: EventTransactionBuffer.java From canal with Apache License 2.0

4 votes

private boolean isDml(EventType eventType) {
    return eventType == EventType.INSERT || eventType == EventType.UPDATE || eventType == EventType.DELETE;
}