Java Code Examples for com.alibaba.otter.canal.protocol.Message#getEntries()
The following examples show how to use
com.alibaba.otter.canal.protocol.Message#getEntries() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: BaseCanalClientTest.java From canal-1.1.3 with Apache License 2.0 | 6 votes |
|
protected void printSummary(Message message, long batchId, int size) {
long memsize = 0;
for (Entry entry : message.getEntries()) {
memsize += entry.getHeader().getEventLength();
}
String startPosition = null;
String endPosition = null;
if (!CollectionUtils.isEmpty(message.getEntries())) {
startPosition = buildPositionForDump(message.getEntries().get(0));
endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
}
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
endPosition });
}
Example 2
| Source File: CanalReaderMessageDumper.java From DataLink with Apache License 2.0 | 6 votes |
|
public static void dumpMessages(Message message, long batchId, int size) {
long memsize = 0;
for (CanalEntry.Entry entry : message.getEntries()) {
memsize += entry.getHeader().getEventLength();
}
String startPosition = null;
String endPosition = null;
if (message.getEntries()!=null && message.getEntries().size()>0) {
startPosition = buildPositionForDump(message.getEntries()
.get(0));
endPosition = buildPositionForDump(message.getEntries().get(
message.getEntries().size() - 1));
}
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
logger.info(context_format, new Object[] { batchId, size, memsize,
format.format(new Date()), startPosition, endPosition });
}
Example 3
| Source File: SelectorTask.java From canal-elasticsearch with Apache License 2.0 | 6 votes |
|
private void printSummary(Message message, long batchId, int size) {
long memsize = 0;
for (CanalEntry.Entry entry : message.getEntries()) {
memsize += entry.getHeader().getEventLength();
}
String startPosition = null;
String endPosition = null;
if (!CollectionUtils.isEmpty(message.getEntries())) {
startPosition = buildPositionForDump(message.getEntries().get(0));
endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
}
logger.info(context_format, new Object[]{batchId, size, memsize, format.format(new Date()), startPosition,
endPosition});
}
Example 4
| Source File: CanalExecutor.java From search-commons with Apache License 2.0 | 6 votes |
|
/**
* 消费当前消息
*/
private void consumerMessage(Message message) {
log.debug("canal instance: " + handle.instanceName() + " get message entry size " + message.getEntries().size());
try {
for (CanalEntry.Entry e : message.getEntries()) {
if (e.getEntryType() != CanalEntry.EntryType.ROWDATA || !e.hasStoreValue()) continue;
CanalEntry.Header header = e.getHeader();
if (header.getExecuteTime() < startRtTime
|| header.getEventType().getNumber() > CanalEntry.EventType.DELETE_VALUE
|| !handle.startHandle(header)) continue;
try {
CanalEntry.RowChange rowChange = CanalEntry.RowChange.parseFrom(e.getStoreValue());
if (rowChange.getIsDdl()) continue;
handle.rowChangeHandle(rowChange);
} catch (InvalidProtocolBufferException e1) {
log.error("canal instance: " + handle.instanceName() + " parse store value have exception: ", e1);
}
}
handle.ack(message.getId());
} finally {
handle.finishMessageHandle();
}
}
Example 5
| Source File: BaseCanalClientTest.java From canal with Apache License 2.0 | 6 votes |
|
protected void printSummary(Message message, long batchId, int size) {
long memsize = 0;
for (Entry entry : message.getEntries()) {
memsize += entry.getHeader().getEventLength();
}
String startPosition = null;
String endPosition = null;
if (!CollectionUtils.isEmpty(message.getEntries())) {
startPosition = buildPositionForDump(message.getEntries().get(0));
endPosition = buildPositionForDump(message.getEntries().get(message.getEntries().size() - 1));
}
SimpleDateFormat format = new SimpleDateFormat(DATE_FORMAT);
logger.info(context_format, new Object[] { batchId, size, memsize, format.format(new Date()), startPosition,
endPosition });
}
Example 6
| Source File: AbstractBasicMessageTransponder.java From spring-boot-starter-canal with MIT License | 5 votes |
|
@Override
protected void distributeEvent(Message message) {
List<CanalEntry.Entry> entries = message.getEntries();
for (CanalEntry.Entry entry : entries) {
//ignore the transaction operations
List<CanalEntry.EntryType> ignoreEntryTypes = getIgnoreEntryTypes();
if (ignoreEntryTypes != null
&& ignoreEntryTypes.stream().anyMatch(t -> entry.getEntryType() == t)) {
continue;
}
CanalEntry.RowChange rowChange;
try {
rowChange = CanalEntry.RowChange.parseFrom(entry.getStoreValue());
} catch (Exception e) {
throw new CanalClientException("ERROR ## parser of event has an error , data:" + entry.toString(),
e);
}
//ignore the ddl operation
if (rowChange.hasIsDdl() && rowChange.getIsDdl()) {
processDdl(rowChange);
continue;
}
for (CanalEntry.RowData rowData : rowChange.getRowDatasList()) {
//distribute to listener interfaces
distributeByImpl(rowChange.getEventType(), rowData);
//distribute to annotation listener interfaces
distributeByAnnotation(destination,
entry.getHeader().getSchemaName(),
entry.getHeader().getTableName(),
rowChange.getEventType(),
rowData);
}
}
}
Example 7
| Source File: MysqlMessageProcessor.java From DBus with Apache License 2.0 | 5 votes |
|
@Override
public List<IGenericMessage> unwrapMessages(byte[] data) throws IOException {
// canal-1.0.24
//List<IGenericMessage> list = new ArrayList<>();
//
//CanalPacket.Messages cMessage = CanalPacket.Messages.parseFrom(data);
//List<ByteString> strings = cMessage.getMessagesList();
//for (ByteString str : strings) {
// CanalEntry.Entry ent = CanalEntry.Entry.parseFrom(str);
// MysqlGenericMessage message = new MysqlGenericMessage(ent);
//
// list.add(message);
//}
// canal-1.1.4
Message message = CanalMessageDeserializer.deserializer(data);
List<IGenericMessage> list = new ArrayList<>();
for (CanalEntry.Entry entry : message.getEntries()) {
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN || entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
logger.debug("the entry type is transaction begin or transaction end.");
continue;
}
//处理带正则的表名
String schemaName = entry.getHeader().getSchemaName();
String tableName = entry.getHeader().getTableName();
String localTable = getLocalTable(schemaName, tableName);
logger.debug("收到表数据{}.{}(基准表名{})数据", schemaName, tableName, localTable);
if (!StringUtils.equals(tableName, localTable)) {
String finalTable = StringUtils.join(new String[]{localTable, tableName}, ".");
CanalEntry.Header header = CanalEntry.Header.newBuilder(entry.getHeader()).setTableName(finalTable).build();
entry = CanalEntry.Entry.newBuilder(entry).setHeader(header).build();
logger.debug("重新生成entry");
}
MysqlGenericMessage mysqlGenericMessage = new MysqlGenericMessage(entry);
list.add(mysqlGenericMessage);
}
return list;
}
Example 8
| Source File: TotoroTransForm.java From canal-elasticsearch with Apache License 2.0 | 5 votes |
|
@Override
public ElasticsearchMetadata trans(Message message) {
List<CanalEntry.Entry> entries = message.getEntries();
ElasticsearchMetadata elasticsearchMetadata = TotoroObjectPool.esMetadata();
elasticsearchMetadata.setBatchId(message.getId());
elasticsearchMetadata.setTransForm(this);
if (entries != null && entries.size() > 0) {
EsEntryArrayList esEntryList = TotoroObjectPool.esEntryArrayList();
entries.forEach(entry -> {
if (messageFilterChain.filter(entry)) {
try {
ElasticsearchMetadata.EsEntry esEntry = getElasticsearchMetadata(entry);
esEntryList.add(esEntry);
} catch (InvalidProtocolBufferException e) {
e.printStackTrace();
}
}
});
elasticsearchMetadata.setEsEntries(esEntryList);
}
logger.info("Trans form complete message id =====> {}", message.getId());
if (logger.isDebugEnabled()) {
logger.debug("Trans form complete elasticsearch metadata =====> {}", elasticsearchMetadata.toString());
}
return elasticsearchMetadata;
}
Example 9
| Source File: CanalReader.java From DataLink with Apache License 2.0 | 3 votes |
|
/**
* <p>
* 设计之初，firstEntryTime的获取方式为：message.getEntries().get(0).getHeader().getExecuteTime()
* 这样获取不够精确，在有大事务或者长事务时，按此取到的firstEntryTime和事务最终提交的时间会有比较大的差距，
* 此时，我们通过firstEntryTime去计算延迟时间的时候，算出来的值就会比较大而触发报警。
* <p>
* 为什么按此取到的firstEntryTime和事务最终提交的时间会有比较大的差距呢？是因为mysql对于每个操作事件，
* 以其真正发生的时间作为ExecuteTime，不会因为在一个事务里，而以提交时间为准，如下所示：
* TRANSACTIONBEGIN 2018-01-01 10:00:00
* INSERT 2018-01-01 10:01:00
* UPDATE 2018-01-01 10:02:00
* TRANSACTIONEND 2018-01-01 10:03:00
* </p>
* <p>
* <p>
* 而应该以第一个TRANSACTIONEND的时间为准，因为TRANSACTIONEND对应的是最终的commit，代表了实际发生
* 如果此批次没有TRANSACTIONEND，说明该批次所有数据都在一个事务里，则以最后一条的时间为准
* 这么改，主要是解决了长事务的问题，此处所说的长事务，是指一个事务从begin到end的跨度比较长，但中间不一定
* 有非常多的操做，并且这些操作的间隔时间比较长。
* </p>
*
* @param message
* @return
*/
private long calcFirstEntryTime(Message message) {
CanalEntry.Entry lastEntry = null;
for (CanalEntry.Entry entry : message.getEntries()) {
lastEntry = entry;
if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONEND) {
break;
}
}
return lastEntry.getHeader().getExecuteTime();
}
