Java Code Examples for com.alibaba.otter.canal.protocol.CanalEntry.Entry

The following examples show how to use com.alibaba.otter.canal.protocol.CanalEntry.Entry. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: canal   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
protected boolean consumeTheEventAndProfilingIfNecessary(List<CanalEntry.Entry> entrys) throws CanalSinkException,
                                                                                       InterruptedException {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }

    boolean result = eventSink.sink(entrys, (runningInfo == null) ? null : runningInfo.getAddress(), destination);

    if (enabled) {
        this.processingInterval = System.currentTimeMillis() - startTs;
    }

    if (consumedEventCount.incrementAndGet() < 0) {
        consumedEventCount.set(0);
    }

    return result;
}
 
Example 2
Source Project: canal-1.1.3   Source File: EntryEventSink.java    License: Apache License 2.0 6 votes vote down vote up
protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
 
Example 3
Source Project: canal   Source File: LogEventConvert.java    License: Apache License 2.0 6 votes vote down vote up
private Entry parseRowsQueryEvent(RowsQueryLogEvent event) {
    if (filterQueryDml) {
        return null;
    }
    // mysql5.6支持,需要设置binlog-rows-query-log-events=1,可详细打印原始DML语句
    String queryString = null;
    try {
        queryString = new String(event.getRowsQuery().getBytes(ISO_8859_1), charset.name());
        String tableName = null;
        if (useDruidDdlFilter) {
            List<DdlResult> results = DruidDdlParser.parse(queryString, null);
            if (results.size() > 0) {
                tableName = results.get(0).getTableName();
            }
        }

        return buildQueryEntry(queryString, event.getHeader(), tableName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}
 
Example 4
Source Project: canal   Source File: BaseCanalClientTest.java    License: Apache License 2.0 6 votes vote down vote up
protected void printSummary(Message message, long batchId, int size) {
    long memsize = 0;
    for (Entry entry : message.getEntries()) {
        memsize += entry.getHeader().getEventLength();
    }

    String startPosition = null;
    String endPosition = null;
    if (!CollectionUtils.isEmpty(message.getEntries())) {
        startPosition = buildPositionForDump(message.getEntries().get(0));
        endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
    }

    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
            endPosition });
}
 
Example 5
Source Project: canal   Source File: EntryEventSink.java    License: Apache License 2.0 6 votes vote down vote up
protected boolean doFilter(CanalEntry.Entry entry) {
    if (filter != null && entry.getEntryType() == EntryType.ROWDATA) {
        String name = getSchemaNameAndTableName(entry);
        boolean need = filter.filter(name);
        if (!need) {
            logger.debug("filter name[{}] entry : {}:{}",
                name,
                entry.getHeader().getLogfileName(),
                entry.getHeader().getLogfileOffset());
        }

        return need;
    } else {
        return true;
    }
}
 
Example 6
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 
Example 7
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
protected LogPosition buildLastPosition(CanalEntry.Entry entry) { // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    // set gtid
    position.setGtid(entry.getHeader().getGtid());

    logPosition.setPostion(position);

    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
 
Example 8
Source Project: DBus   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
public AbstractEventParser(){
    // 初始化一下
    transactionBuffer = new EventTransactionBuffer(new TransactionFlushCallback() {

        public void flush(List<CanalEntry.Entry> transaction) throws InterruptedException {
            boolean successed = consumeTheEventAndProfilingIfNecessary(transaction);
            if (!running) {
                return;
            }

            if (!successed) {
                throw new CanalParseException("consume failed!");
            }

            LogPosition position = buildLastTransactionPosition(transaction);
            if (position != null) { // 可能position为空
                logPositionManager.persistLogPosition(AbstractEventParser.this.destination, position);
            }
        }
    });
}
 
Example 9
Source Project: DBus   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 
Example 10
Source Project: canal   Source File: AbstractEventParser.java    License: Apache License 2.0 6 votes vote down vote up
protected CanalEntry.Entry parseAndProfilingIfNecessary(EVENT bod, boolean isSeek) throws Exception {
    long startTs = -1;
    boolean enabled = getProfilingEnabled();
    if (enabled) {
        startTs = System.currentTimeMillis();
    }
    CanalEntry.Entry event = binlogParser.parse(bod, isSeek);
    if (enabled) {
        this.parsingInterval = System.currentTimeMillis() - startTs;
    }

    if (parsedEventCount.incrementAndGet() < 0) {
        parsedEventCount.set(0);
    }
    return event;
}
 
Example 11
private void publishCanalEvent(Entry entry) {
    EventType eventType = entry.getHeader().getEventType();
    switch (eventType) {
        case INSERT:
            applicationContext.publishEvent(new InsertAbstractCanalEvent(entry));
            break;
        case UPDATE:
            applicationContext.publishEvent(new UpdateAbstractCanalEvent(entry));
            break;
        case DELETE:
            applicationContext.publishEvent(new DeleteAbstractCanalEvent(entry));
            break;
        default:
            break;
    }
}
 
Example 12
@Override
public void onApplicationEvent(EVENT event) {
    Entry entry = event.getEntry();
    String database = entry.getHeader().getSchemaName();
    String table = entry.getHeader().getTableName();
    IndexTypeModel indexTypeModel = mappingService.getIndexType(new DatabaseTableModel(database, table));
    if (indexTypeModel == null) {
        return;
    }
    String index = indexTypeModel.getIndex();
    String type = indexTypeModel.getType();
    RowChange change;
    try {
        change = RowChange.parseFrom(entry.getStoreValue());
    } catch (InvalidProtocolBufferException e) {
        logger.error("canalEntry_parser_error,根据CanalEntry获取RowChange失败!", e);
        return;
    }
    change.getRowDatasList().forEach(rowData -> doSync(database, table, index, type, rowData));
}
 
Example 13
Source Project: canal-1.1.3   Source File: BaseCanalClientTest.java    License: Apache License 2.0 5 votes vote down vote up
protected String buildPositionForDump(Entry entry) {
    long time = entry.getHeader().getExecuteTime();
    Date date = new Date(time);
    SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
    String position = entry.getHeader().getLogfileName() + ":" + entry.getHeader().getLogfileOffset() + ":"
                      + entry.getHeader().getExecuteTime() + "(" + format.format(date) + ")";
    if (StringUtils.isNotEmpty(entry.getHeader().getGtid())) {
        position += " gtid(" + entry.getHeader().getGtid() + ")";
    }
    return position;
}
 
Example 14
Source Project: canal   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry buildQueryEntry(String queryString, LogHeader logHeader, String tableName) {
    Header header = createHeader(logHeader, "", tableName, EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
 
Example 15
Source Project: canal   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry parseRandLogEvent(RandLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 
Example 16
Source Project: canal   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
public void setFieldFilterMap(Map<String, List<String>> fieldFilterMap) {
   	if (fieldFilterMap != null) {
   		this.fieldFilterMap = fieldFilterMap;
   	} else {
   		this.fieldFilterMap = new HashMap<String, List<String>>();
   	}
	
	
	for (Map.Entry<String, List<String>> entry : this.fieldFilterMap.entrySet()) {
		logger.warn("--> init field filter : " + entry.getKey() + "->" + entry.getValue());
	}
}
 
Example 17
Source Project: canal   Source File: AbstractEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
 
Example 18
Source Project: canal-1.1.3   Source File: ProtocolTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimple() throws IOException {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName("mysql-bin.000001");
    headerBuilder.setLogfileOffset(1024);
    headerBuilder.setExecuteTime(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.ROWDATA);
    Entry entry = entryBuilder.build();
    Message message = new Message(3, true, Arrays.asList(entry.toByteString()));

    byte[] body = buildData(message);
    Packet packet = Packet.parseFrom(body);
    switch (packet.getType()) {
        case MESSAGES: {
            if (!packet.getCompression().equals(Compression.NONE)) {
                throw new CanalClientException("compression is not supported in this connector");
            }

            Messages messages = Messages.parseFrom(packet.getBody());
            Message result = new Message(messages.getBatchId());
            for (ByteString byteString : messages.getMessagesList()) {
                result.addEntry(Entry.parseFrom(byteString));
            }

            System.out.println(result);
            break;
        }
        default: {
            throw new CanalClientException("unexpected packet type: " + packet.getType());
        }
    }
}
 
Example 19
Source Project: canal-1.1.3   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry parseHeartbeatLogEvent(HeartbeatLogEvent logEvent) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setEventType(EventType.MHEARTBEAT);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    entryBuilder.setEntryType(EntryType.HEARTBEAT);
    return entryBuilder.build();
}
 
Example 20
Source Project: canal-1.1.3   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry parseUserVarLogEvent(UserVarLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 
Example 21
Source Project: canal-1.1.3   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry parseIntrvarLogEvent(IntvarLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}
 
Example 22
Source Project: canal   Source File: AbstractEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常,或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中,目前不会提交到store中,会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed ", e);
            }
        }

    };
}
 
Example 23
Source Project: canal-1.1.3   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry buildQueryEntry(String queryString, LogHeader logHeader, String tableName) {
    Header header = createHeader(logHeader, "", tableName, EventType.QUERY);
    RowChange.Builder rowChangeBuider = RowChange.newBuilder();
    rowChangeBuider.setSql(queryString);
    rowChangeBuider.setEventType(EventType.QUERY);
    return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
}
 
Example 24
Source Project: canal   Source File: MemoryEventStoreBase.java    License: Apache License 2.0 5 votes vote down vote up
protected Event buildEvent(String binlogFile, long offset, long timestamp) {
    Header.Builder headerBuilder = Header.newBuilder();
    headerBuilder.setLogfileName(binlogFile);
    headerBuilder.setLogfileOffset(offset);
    headerBuilder.setExecuteTime(timestamp);
    headerBuilder.setEventLength(1024);
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(headerBuilder.build());
    Entry entry = entryBuilder.build();

    return new Event(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L), entry);
}
 
Example 25
Source Project: canal-1.1.3   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
public static Entry createEntry(Header header, EntryType entryType, ByteString storeValue) {
    Entry.Builder entryBuilder = Entry.newBuilder();
    entryBuilder.setHeader(header);
    entryBuilder.setEntryType(entryType);
    entryBuilder.setStoreValue(storeValue);
    return entryBuilder.build();
}
 
Example 26
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected LogPosition buildLastTransactionPosition(List<CanalEntry.Entry> entries) { // 初始化一下
    for (int i = entries.size() - 1; i > 0; i--) {
        CanalEntry.Entry entry = entries.get(i);
        if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {// 尽量记录一个事务做为position
            return buildLastPosition(entry);
        }
    }

    return null;
}
 
Example 27
Source Project: canal-1.1.3   Source File: AbstractEventParser.java    License: Apache License 2.0 5 votes vote down vote up
protected TimerTask buildHeartBeatTimeTask(ErosaConnection connection) {
    return new TimerTask() {

        public void run() {
            try {
                if (exception == null || lastEntryTime > 0) {
                    // 如果未出现异常,或者有第一条正常数据
                    long now = System.currentTimeMillis();
                    long inteval = (now - lastEntryTime) / 1000;
                    if (inteval >= detectingIntervalInSeconds) {
                        Header.Builder headerBuilder = Header.newBuilder();
                        headerBuilder.setExecuteTime(now);
                        Entry.Builder entryBuilder = Entry.newBuilder();
                        entryBuilder.setHeader(headerBuilder.build());
                        entryBuilder.setEntryType(EntryType.HEARTBEAT);
                        Entry entry = entryBuilder.build();
                        // 提交到sink中,目前不会提交到store中,会在sink中进行忽略
                        consumeTheEventAndProfilingIfNecessary(Arrays.asList(entry));
                    }
                }

            } catch (Throwable e) {
                logger.warn("heartBeat run failed ", e);
            }
        }

    };
}
 
Example 28
Source Project: canal   Source File: Message.java    License: Apache License 2.0 5 votes vote down vote up
public Message(long id, boolean raw, List entries){
    this.id = id;
    if (raw) {
        this.rawEntries = entries == null ? new ArrayList<ByteString>() : entries;
    } else {
        this.entries = entries == null ? new ArrayList<Entry>() : entries;
    }
    this.raw = raw;
}
 
Example 29
Source Project: canal-1.1.3   Source File: GroupEventPaserTest.java    License: Apache License 2.0 5 votes vote down vote up
private BinlogParser buildParser(AuthenticationInfo info) {
    return new AbstractBinlogParser<LogEvent>() {

        @Override
        public Entry parse(LogEvent event, boolean isSeek) throws CanalParseException {
            return null;
        }
    };
}
 
Example 30
Source Project: canal   Source File: LogEventConvert.java    License: Apache License 2.0 5 votes vote down vote up
private Entry parseIntrvarLogEvent(IntvarLogEvent event) {
    if (filterQueryDml) {
        return null;
    }

    return buildQueryEntry(event.getQuery(), event.getHeader());
}