com.github.shyiko.mysql.binlog.event.Event Java Examples

@Override
protected EventBaseDTO formatData(Event event) {
    UpdateRowsEventData d = event.getData();
    UpdateRowsDTO updateRowsDTO = new UpdateRowsDTO();
    updateRowsDTO.setEventType(DatabaseEvent.UPDATE_ROWS);
    //添加表信息
    ColumnsTableMapEventData tableMapData = context.getTableMapData(d.getTableId());
    updateRowsDTO.setDatabase(tableMapData.getDatabase());
    updateRowsDTO.setTable(tableMapData.getTable());
    updateRowsDTO.setNamespace(context.getBinaryLogConfig().getNamespace());
    //添加列映射
    int[] includedColumns = d.getIncludedColumns().stream().toArray();
    List<UpdateRow> urs = d.getRows().stream()
            .map(e -> new UpdateRow(convert(e.getKey(),includedColumns,tableMapData),
                    convert(e.getValue(),includedColumns,tableMapData))).collect(Collectors.toList());
    updateRowsDTO.setRows(urs);
    return updateRowsDTO;
}
 

@Test
public void testTableMapEvent() {
  eventHeader.setEventType(EventType.TABLE_MAP);
  TableMapEventData eventData = new TableMapEventData();
  eventData.setDatabase(DATABASE);
  eventData.setTable(TABLE);
  eventData.setTableId(TABLE_ID);
  eventData.setColumnTypes(new byte[] {(byte) 0, (byte) 1, (byte) 2});

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof TableMapEvent);
  TableMapEvent tableMapEvent = (TableMapEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, tableMapEvent.getBinlogFilePos());
  assertEquals(DATABASE, tableMapEvent.getDatabase());
  assertEquals(TABLE, tableMapEvent.getTable());
  assertEquals(TABLE_ID, tableMapEvent.getTableId());
  assertEquals(
      ImmutableList.of(ColumnDataType.DECIMAL, ColumnDataType.TINY, ColumnDataType.SHORT),
      tableMapEvent.getColumnTypes());
}
 

@Test
public void testDeleteEvent() {
  eventHeader.setEventType(EventType.EXT_DELETE_ROWS);
  DeleteRowsEventData eventData = new DeleteRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(PREV_ROW));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof DeleteEvent);
  DeleteEvent deleteEvent = (DeleteEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, deleteEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(PREV_ROW), deleteEvent.getRows());
  assertEquals(SERVER_ID, deleteEvent.getServerId());
  assertEquals(TABLE_ID, deleteEvent.getTableId());
  assertEquals(TIMESTAMP, deleteEvent.getTimestamp());
}
 

@Test
public void testUpdateEvent() {
  eventHeader.setEventType(EventType.EXT_UPDATE_ROWS);
  UpdateRowsEventData eventData = new UpdateRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(Maps.immutableEntry(PREV_ROW, ROW)));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof UpdateEvent);
  UpdateEvent updateEvent = (UpdateEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, updateEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(Maps.immutableEntry(PREV_ROW, ROW)), updateEvent.getRows());
  assertEquals(SERVER_ID, updateEvent.getServerId());
  assertEquals(TABLE_ID, updateEvent.getTableId());
  assertEquals(TIMESTAMP, updateEvent.getTimestamp());
}
 

@Test
public void testWriteEvent() {
  eventHeader.setEventType(EventType.EXT_WRITE_ROWS);
  WriteRowsEventData eventData = new WriteRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(ROW));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof WriteEvent);
  WriteEvent writeEvent = (WriteEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, writeEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(ROW), writeEvent.getRows());
  assertEquals(SERVER_ID, writeEvent.getServerId());
  assertEquals(TABLE_ID, writeEvent.getTableId());
  assertEquals(TIMESTAMP, writeEvent.getTimestamp());
}
 

public void onEvent(Event event) {
  Preconditions.checkState(isStarted(), "Source is not started and should not process events");

  final EventHeaderV4 header = event.getHeader();
  final BinlogFilePos filePos =
      new BinlogFilePos(
          binlogClient.getBinlogFilename(),
          header.getPosition(),
          header.getNextPosition(),
          binlogClient.getGtidSet(),
          serverUUID);

  BinaryLogConnectorEventMapper.INSTANCE
      .map(event, filePos)
      .ifPresent(BinaryLogConnectorSource.super::processEvent);
}
 

@Test
public void testXidEvent() {
  long xid = 88888L;
  eventHeader.setEventType(EventType.XID);
  XidEventData eventData = new XidEventData();
  eventData.setXid(xid);

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof XidEvent);
  XidEvent xidEvent = (XidEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, xidEvent.getBinlogFilePos());
  assertEquals(SERVER_ID, xidEvent.getServerId());
  assertEquals(TIMESTAMP, xidEvent.getTimestamp());
  assertEquals(xid, xidEvent.getXid());
}
 

@Test
public void testQueryEvent() {
  String sql = "CREATE UNIQUE INDEX unique_index ON `my_db`.`my_table` (`col1`, `col2`)";
  eventHeader.setEventType(EventType.QUERY);
  QueryEventData eventData = new QueryEventData();
  eventData.setDatabase(DATABASE);
  eventData.setSql(sql);

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof QueryEvent);
  QueryEvent queryEvent = (QueryEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, queryEvent.getBinlogFilePos());
  assertEquals(DATABASE, queryEvent.getDatabase());
  assertEquals(SERVER_ID, queryEvent.getServerId());
  assertEquals(TIMESTAMP, queryEvent.getTimestamp());
  assertEquals(sql, queryEvent.getSql());
}
 

@Override
public void handle(Event event) {
    TableMapEventData d = event.getData();
    log.debug("TableMapEventData:{}", d);
    ColumnsTableMapEventData tableMapEventData = context.getTableMapData(d.getTableId());
    //如果表结构有变化，重新设置映射信息
    if (tableMapEventData == null || !ColumnsTableMapEventData.checkEqual(d, tableMapEventData)) {
        log.info("更新表映射：{}-{}", d.getDatabase(), d.getTable());
        ColumnsTableMapEventData data = new ColumnsTableMapEventData(d);
        String sql = "show columns from `" + d.getTable() + "` from `" + d.getDatabase() + "`";
        try (Connection conn = context.getDataSource().getConnection();
             PreparedStatement ps = conn.prepareStatement(sql);
             ResultSet resultSet = ps.executeQuery();) {
            while (resultSet.next()) {
                data.addColumnName(resultSet.getString("Field"));
            }
            //将表id和表映射
            context.addTableMapData(d.getTableId(), data);
        } catch (SQLException e) {
            log.error("获取表数据错误,sql语句为{}，异常如下:{}", sql, e);
        }
    }
}
 

@Override
protected EventBaseDTO formatData(Event event) {
    WriteRowsEventData d = event.getData();
    WriteRowsDTO writeRowsDTO = new WriteRowsDTO();
    writeRowsDTO.setEventType(DatabaseEvent.WRITE_ROWS);
    //添加表信息
    ColumnsTableMapEventData tableMapData = context.getTableMapData(d.getTableId());
    writeRowsDTO.setDatabase(tableMapData.getDatabase());
    writeRowsDTO.setTable(tableMapData.getTable());
    writeRowsDTO.setNamespace(context.getBinaryLogConfig().getNamespace());
    //添加列映射
    int[] includedColumns = d.getIncludedColumns().stream().toArray();
    writeRowsDTO.setRowMaps(d.getRows().stream()
            .map(r -> convert(r,includedColumns,tableMapData)).collect(Collectors.toList()));
    return writeRowsDTO;
}
 

private void processXidEvent(Event event) {
    EventHeaderV4 header = event.getHeader();
    XidEventData data = event.getData();

    String binlogFilename = binaryLogClient.getBinlogFilename();
    Long position = header.getNextPosition();
    Long xid = data.getXid();

    BinlogPosition binlogPosition = new BinlogPosition(binlogFilename, position);
    transaction.setNextBinlogPosition(binlogPosition);
    transaction.setXid(xid);

    replicator.commit(transaction, true);

    transaction = new Transaction(config);
}
 

FilterRes decide(SimpleEventType simpleEventType, BinlogDataId dataId, Event[] events) {
  if (logger.isTraceEnabled()) {
    logger.trace("Receive binlog event: {}", Arrays.toString(events));
  }
  SyncData[] aim = toSyncData(simpleEventType, dataId, events[0], events[1]);
  if (aim == null) { // not interested in this database+table
    return FilterRes.DENY;
  }

  boolean output = producerSink.output(aim);
  return output ? FilterRes.ACCEPT : FilterRes.DENY;
}
 

private void processTableMapEvent(Event event) {
    TableMapEventData data = event.getData();
    String dbName = data.getDatabase();
    String tableName = data.getTable();
    Long tableId = data.getTableId();

    Table table = schema.getTable(dbName, tableName);
    if (table != null){
        tableMap.put(tableId, table);
    }
}
 

@Test
public void testIgnoredEvents() {
  eventHeader.setEventType(EventType.UNKNOWN);
  XAPrepareEventData eventData = new XAPrepareEventData();

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertFalse(binlogEvent.isPresent());
}
 

private SyncData[] toSyncData(SimpleEventType type, BinlogDataId dataId, Event... e) {
  TableMapEventData tableMap = e[0].getData();
  TableMeta table = consumerSchemaMeta.findTable(tableMap.getDatabase(), tableMap.getTable());
  if (table == null) {
    return null;
  }

  List<IndexedFullRow> indexedRow = RowsEvent
      .getIndexedRows(type, e[1].getData(), table.getPrimaryKeys());
  List<NamedFullRow> namedRow = RowsEvent
      .getNamedRows(indexedRow, table.getInterestedAndPkIndex(), table.getInterestedAndPkIndexToName());
  String primaryKey = RowsEvent.getPrimaryKey(table.getInterestedAndPkIndexToName(), table.getPrimaryKeys());
  SyncData[] res = new SyncData[namedRow.size()];
  boolean hasData = false;
  for (int i = 0; i < res.length; i++) {
    NamedFullRow row = namedRow.get(i);
    if (onlyUpdated && type == SimpleEventType.UPDATE && row.getUpdated().isEmpty()) {
      logger.debug("Discard {} because [{}]", dataId.eventId(), row);
      // even though in one update event, multiple rows can have different updated column,
      // so we can only skip one by one
      // e.g. we listening 'name1' but not 'name2' and the following update will make all updates in a single event
      // UPDATE relation
      // SET name1 = CASE WHEN userid1 = 3 THEN 'jack' ELSE name1 END,
      //     name2 = CASE WHEN userid2 = 3 THEN 'jack' ELSE name2 END
      continue;
    }
    Object pk = row.get(primaryKey);
    if (!table.isInterestedPK()) {
      row.remove(primaryKey);
    }
    hasData = true;
    res[i] = new SyncData(dataId.copyAndSetOrdinal(i), type, tableMap.getDatabase(), tableMap.getTable(), primaryKey, pk, row);
  }
  if (hasData) {
    return res;
  }
  return null;
}
 

@Test
public void testFormatDescriptionEvent() {
  eventHeader.setEventType(EventType.FORMAT_DESCRIPTION);
  FormatDescriptionEventData eventData = new FormatDescriptionEventData();

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof StartEvent);
  StartEvent startEvent = (StartEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, startEvent.getBinlogFilePos());
  assertEquals(SERVER_ID, startEvent.getServerId());
  assertEquals(TIMESTAMP, startEvent.getTimestamp());
}
 

@Override
public boolean dispatch(SimpleEventType simpleEventType, Object... data) {
  Preconditions.checkState(data.length == 2);
  Event[] events = new Event[]{(Event) data[0], (Event) data[1]};
  BinlogDataId dataId = DataId.fromEvent(events, binlogInfo.get().getBinlogFilename());
  MDC.put(LogbackLoggingField.EID, dataId.eventId());
  boolean res = true;
  for (ConsumerChannel consumerChannel : consumerChannels) {
    FilterRes decide = consumerChannel.decide(simpleEventType, dataId, events);
    res = res && FilterRes.ACCEPT == decide;
  }
  MDC.remove(LogbackLoggingField.EID);
  return res;
}
 

/**
 * <a href="https://github.com/shyiko/mysql-binlog-connector-java/issues/200">binlog table
 * map</a>
 */
static BinlogDataId fromEvent(Event[] event, String binlogFileName) {
  EventHeaderV4 tableMap = event[0].getHeader();
  EventHeaderV4 second = event[1].getHeader();
  // have to remember table map event for restart
  // have to add data event position for unique id:
  // because one table map event may follow multiple data event
  return new BinlogDataId(binlogFileName, tableMap.getPosition(), second.getPosition());
}
 

public RawEventInvocationHandler(BinaryLogClient binaryLogClient, Event event) {
    this.gtidSet = binaryLogClient.getGtidSet();
    this.event = event;
    this.eventDataSubTypes = Stream
            .of(RawEventType.values())
            .map(RawEventType::getDefinition)
            .distinct()
            .collect(
                    Collectors.toMap(
                        (value) -> value.getSimpleName().replace("Raw", "").toLowerCase(),
                        (value) -> value
                    )
            );
}
 

private SourceOffset createOffset(Event event) {
  if (isGtidEnabled()) {
    return new GtidSourceOffset(currentGtidSet)
        .withIncompleteTransaction(currentTxGtid, currentTxEventSeqNo);
  } else {
    return new BinLogPositionSourceOffset(
        currentBinLogFileName,
        ((EventHeaderV4) event.getHeader()).getNextPosition()
    );
  }
}
 

private SourceOffset createOffset(Event event) {
  if (isGtidEnabled()) {
    return new GtidSourceOffset(currentGtidSet)
        .withIncompleteTransaction(currentTxGtid, currentTxEventSeqNo);
  } else {
    return new BinLogPositionSourceOffset(
        currentBinLogFileName,
        ((EventHeaderV4) event.getHeader()).getNextPosition()
    );
  }
}
 

@Override
public void onEvent(Event event) {
    while (!stopNow.get()) {
        RawBinlogEvent ep = new RawBinlogEvent(event, client.getBinlogFilename());
        try {
            if (queue.offer(ep, QUEUE_OFFER_TIMEOUT_MSEC, TimeUnit.MILLISECONDS)) {
                return;
            } else {
                throw new RuntimeException("Unable to add event to the queue");
            }
        } catch (InterruptedException e) {
            throw new RuntimeException("Interrupted while adding event to the queue");
        }
    }
}
 

@Override
public void handle(Event event) {
    boolean ddleventEnable = Boolean.parseBoolean(System.getProperty(DDLEVENT_ENABLE_KEY,DDLEVENT_ENABLE));
    if(ddleventEnable){
        QueryEventData data = event.getData();
        String sql = data.getSql();
        if (sql.contains(SUBSCRIBE_SQL_EVENT)) {
            log.info("数据库：{}发生alter table事件", data.getDatabase());
            String topic = System.getProperty(DEFAULT_TOPIC_KEY,DEFAULT_TOPIC);
            dataPublisher.pushToKafka(topic, data);
        }
    }
    updateBinaryLogStatus(event.getHeader());
}
 

private void processWriteEvent(Event event) {
    WriteRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Serializable[]> list = data.getRows();

    for (Serializable[] row : list) {
        addRow(EntryType.CREATE, tableId, row, null);
    }
}
 

private void processUpdateEvent(Event event) {
    UpdateRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Map.Entry<Serializable[], Serializable[]>> list = data.getRows();

    for (Map.Entry<Serializable[], Serializable[]> entry : list) {
        addRow(EntryType.UPDATE, tableId, entry.getValue(), entry.getKey());
    }
}
 

private void processDeleteEvent(Event event) {
    DeleteRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Serializable[]> list = data.getRows();

    for (Serializable[] row : list) {
        addRow(EntryType.DELETE, tableId, null, row);
    }

}
 

private void processQueryEvent(Event event) {
    QueryEventData data = event.getData();
    String sql = data.getSql();

    if (createTablePattern.matcher(sql).find()) {
        schema.reset();
    }
}
 

@Override
protected Set<ClientInfo> filter(Event event) {
    WriteRowsEventData d = event.getData();
    long tableId = d.getTableId();
    TableMapEventData tableMapEventData = context.getTableMapData(tableId);
    String tableKey = tableMapEventData.getDatabase().concat("/").concat(tableMapEventData.getTable());
    return clientInfoMap.get(tableKey);
}
 

@Override
protected Set<ClientInfo> filter(Event event) {
    UpdateRowsEventData d = event.getData();
    long tableId = d.getTableId();
    TableMapEventData tableMapEventData =  context.getTableMapData(tableId);
    String tableKey = tableMapEventData.getDatabase().concat("/").concat(tableMapEventData.getTable());
    return clientInfoMap.get(tableKey);
}
 

@Override
public void onEvent(Event event) {
    try {
        while (true) {
            if (queue.offer(event, 100, TimeUnit.MILLISECONDS)) {
                return;
            }
        }
    } catch (InterruptedException e) {
        log.error("Mysql task EventListener#onEvent error.", e);
    }
}