com.alibaba.otter.canal.protocol.CanalEntry.Entry Java Examples

protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
                                                                                       InterruptedException {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }

    boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);

    if (enabled) {
        this.processingInterval = System.currentTimeMillis() - startTs;
    }

    if (consumedEventCount.incrementAndGet() < 0) {
        consumedEventCount.set(0);
    }

    return result;
}
 

protected void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition });
}
 

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        String tableName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, null);
            if (results.size() > 0) {
                tableName = results.get(0).getTableName();
            }
        }

        return buildQueryEntry(queryString, event.getHeader(), tableName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}
 

protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    // set gtid
    position.setGtid(entry.getHeader().getGtid());

    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
 

protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 

private void publishCanalEvent(Entry entry) {
    EventType eventType = entry.getHeader().getEventType();
    switch (eventType) {
        case INSERT:
            applicationContext.publishEvent(new InsertAbstractCanalEvent(entry));
            break;
        case UPDATE:
            applicationContext.publishEvent(new UpdateAbstractCanalEvent(entry));
            break;
        case DELETE:
            applicationContext.publishEvent(new DeleteAbstractCanalEvent(entry));
            break;
        default:
            break;
    }
}
 

@Override
public void onApplicationEvent(EVENT event) {
    Entry entry = event.getEntry();
    String database = entry.getHeader().getSchemaName();
    String table = entry.getHeader().getTableName();
    IndexTypeModel indexTypeModel = mappingService.getIndexType(new DatabaseTableModel(database, table));
    if (indexTypeModel == null) {
        return;
    }
    String index = indexTypeModel.getIndex();
    String type = indexTypeModel.getType();
    RowChange change;
    try {
        change = RowChange.parseFrom(entry.getStoreValue());
    } catch (InvalidProtocolBufferException e) {
        logger.error("canalEntry_parser_error,根据CanalEntry获取RowChange失败！", e);
        return;
    }
    change.getRowDatasList().forEach(rowData -> doSync(database, table, index, type, rowData));
}
 

public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        return buildQueryEntry(queryString, event.getHeader());
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}
 

private Entry parseRandLogEvent(RandLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 

private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
 

private Entry parseRandLogEvent(RandLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 

private static Entry buildEntry(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    return entryBuilder.build();
}
 

private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持，需要设置binlog-rows-query-log-events=1，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        return buildQueryEntry(queryString, event.getHeader());
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}
 

private Entry buildQueryEntry(String queryString, LogHeader logHeader) {
    Header header = createHeader(binlogFileName, logHeader, "", "", EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
 

public void setFieldBlackFilterMap(Map<String, List<String>> fieldBlackFilterMap) {
	if (fieldBlackFilterMap != null) {
   		this.fieldBlackFilterMap = fieldBlackFilterMap;
   	} else {
   		this.fieldBlackFilterMap = new HashMap<String, List<String>>();
   	}
	
	for (Map.Entry<String, List<String>> entry : this.fieldBlackFilterMap.entrySet()) {
		logger.warn("--> init field black filter : " + entry.getKey() + "->" + entry.getValue());
	}
}
 

private Entry parseIntrvarLogEvent(IntvarLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 

private Entry parseUserVarLogEvent(UserVarLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 

private Entry parseAnnotateRowsEvent(AnnotateRowsEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mariaDb支持，需要设置binlog_annotate_row_events=true，可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        return buildQueryEntry(queryString, event.getHeader());
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}
 

private Entry parseIntrvarLogEvent(IntvarLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 

protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常，或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中，目前不会提交到store中，会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed " + ExceptionUtils.getStackTrace(e));
            }
        }

    };
}
 

protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
 

protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
 

protected String buildPositionForDump(Entry entry) {
    long time = entry.getHeader().getExecuteTime();
    Date date = new Date(time);
    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    String position = entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":"
                      + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")";
    if (StringUtils.isNotEmpty(entry.getHeader().getGtid())) {
        position += " gtid(" + entry.getHeader().getGtid() + ")";
    }
    return position;
}
 

private static Entry buildEntry(String binlogFile, long offset, long timestamp, EntryType type) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(type);
    return entryBuilder.build();
}
 

protected Event buildEvent(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    headerBuilder.setEventLength(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    Entry entry = entryBuilder.build();

    return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
}