com.github.shyiko.mysql.binlog.event.Event Java Examples

The following examples show how to use com.github.shyiko.mysql.binlog.event.Event. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BinLogUpdateEventHandler.java    From kkbinlog with Apache License 2.0 6 votes vote down vote up
@Override
protected EventBaseDTO formatData(Event event) {
    UpdateRowsEventData d = event.getData();
    UpdateRowsDTO updateRowsDTO = new UpdateRowsDTO();
    updateRowsDTO.setEventType(DatabaseEvent.UPDATE_ROWS);
    //添加表信息
    ColumnsTableMapEventData tableMapData = context.getTableMapData(d.getTableId());
    updateRowsDTO.setDatabase(tableMapData.getDatabase());
    updateRowsDTO.setTable(tableMapData.getTable());
    updateRowsDTO.setNamespace(context.getBinaryLogConfig().getNamespace());
    //添加列映射
    int[] includedColumns = d.getIncludedColumns().stream().toArray();
    List<UpdateRow> urs = d.getRows().stream()
            .map(e -> new UpdateRow(convert(e.getKey(),includedColumns,tableMapData),
                    convert(e.getValue(),includedColumns,tableMapData))).collect(Collectors.toList());
    updateRowsDTO.setRows(urs);
    return updateRowsDTO;
}
 
Example #2
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
@Test
public void testTableMapEvent() {
  eventHeader.setEventType(EventType.TABLE_MAP);
  TableMapEventData eventData = new TableMapEventData();
  eventData.setDatabase(DATABASE);
  eventData.setTable(TABLE);
  eventData.setTableId(TABLE_ID);
  eventData.setColumnTypes(new byte[] {(byte) 0, (byte) 1, (byte) 2});

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof TableMapEvent);
  TableMapEvent tableMapEvent = (TableMapEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, tableMapEvent.getBinlogFilePos());
  assertEquals(DATABASE, tableMapEvent.getDatabase());
  assertEquals(TABLE, tableMapEvent.getTable());
  assertEquals(TABLE_ID, tableMapEvent.getTableId());
  assertEquals(
      ImmutableList.of(ColumnDataType.DECIMAL, ColumnDataType.TINY, ColumnDataType.SHORT),
      tableMapEvent.getColumnTypes());
}
 
Example #3
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
@Test
public void testDeleteEvent() {
  eventHeader.setEventType(EventType.EXT_DELETE_ROWS);
  DeleteRowsEventData eventData = new DeleteRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(PREV_ROW));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof DeleteEvent);
  DeleteEvent deleteEvent = (DeleteEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, deleteEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(PREV_ROW), deleteEvent.getRows());
  assertEquals(SERVER_ID, deleteEvent.getServerId());
  assertEquals(TABLE_ID, deleteEvent.getTableId());
  assertEquals(TIMESTAMP, deleteEvent.getTimestamp());
}
 
Example #4
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
@Test
public void testUpdateEvent() {
  eventHeader.setEventType(EventType.EXT_UPDATE_ROWS);
  UpdateRowsEventData eventData = new UpdateRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(Maps.immutableEntry(PREV_ROW, ROW)));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof UpdateEvent);
  UpdateEvent updateEvent = (UpdateEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, updateEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(Maps.immutableEntry(PREV_ROW, ROW)), updateEvent.getRows());
  assertEquals(SERVER_ID, updateEvent.getServerId());
  assertEquals(TABLE_ID, updateEvent.getTableId());
  assertEquals(TIMESTAMP, updateEvent.getTimestamp());
}
 
Example #5
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteEvent() {
  eventHeader.setEventType(EventType.EXT_WRITE_ROWS);
  WriteRowsEventData eventData = new WriteRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(ROW));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof WriteEvent);
  WriteEvent writeEvent = (WriteEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, writeEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(ROW), writeEvent.getRows());
  assertEquals(SERVER_ID, writeEvent.getServerId());
  assertEquals(TABLE_ID, writeEvent.getTableId());
  assertEquals(TIMESTAMP, writeEvent.getTimestamp());
}
 
Example #6
Source File: BinaryLogConnectorSource.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
public void onEvent(Event event) {
  Preconditions.checkState(isStarted(), "Source is not started and should not process events");

  final EventHeaderV4 header = event.getHeader();
  final BinlogFilePos filePos =
      new BinlogFilePos(
          binlogClient.getBinlogFilename(),
          header.getPosition(),
          header.getNextPosition(),
          binlogClient.getGtidSet(),
          serverUUID);

  BinaryLogConnectorEventMapper.INSTANCE
      .map(event, filePos)
      .ifPresent(BinaryLogConnectorSource.super::processEvent);
}
 
Example #7
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
@Test
public void testXidEvent() {
  long xid = 88888L;
  eventHeader.setEventType(EventType.XID);
  XidEventData eventData = new XidEventData();
  eventData.setXid(xid);

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof XidEvent);
  XidEvent xidEvent = (XidEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, xidEvent.getBinlogFilePos());
  assertEquals(SERVER_ID, xidEvent.getServerId());
  assertEquals(TIMESTAMP, xidEvent.getTimestamp());
  assertEquals(xid, xidEvent.getXid());
}
 
Example #8
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 6 votes vote down vote up
@Test
public void testQueryEvent() {
  String sql = "CREATE UNIQUE INDEX unique_index ON `my_db`.`my_table` (`col1`, `col2`)";
  eventHeader.setEventType(EventType.QUERY);
  QueryEventData eventData = new QueryEventData();
  eventData.setDatabase(DATABASE);
  eventData.setSql(sql);

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof QueryEvent);
  QueryEvent queryEvent = (QueryEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, queryEvent.getBinlogFilePos());
  assertEquals(DATABASE, queryEvent.getDatabase());
  assertEquals(SERVER_ID, queryEvent.getServerId());
  assertEquals(TIMESTAMP, queryEvent.getTimestamp());
  assertEquals(sql, queryEvent.getSql());
}
 
Example #9
Source File: BinLogTableMapEventHandler.java    From kkbinlog with Apache License 2.0 6 votes vote down vote up
@Override
public void handle(Event event) {
    TableMapEventData d = event.getData();
    log.debug("TableMapEventData:{}", d);
    ColumnsTableMapEventData tableMapEventData = context.getTableMapData(d.getTableId());
    //如果表结构有变化,重新设置映射信息
    if (tableMapEventData == null || !ColumnsTableMapEventData.checkEqual(d, tableMapEventData)) {
        log.info("更新表映射:{}-{}", d.getDatabase(), d.getTable());
        ColumnsTableMapEventData data = new ColumnsTableMapEventData(d);
        String sql = "show columns from `" + d.getTable() + "` from `" + d.getDatabase() + "`";
        try (Connection conn = context.getDataSource().getConnection();
             PreparedStatement ps = conn.prepareStatement(sql);
             ResultSet resultSet = ps.executeQuery();) {
            while (resultSet.next()) {
                data.addColumnName(resultSet.getString("Field"));
            }
            //将表id和表映射
            context.addTableMapData(d.getTableId(), data);
        } catch (SQLException e) {
            log.error("获取表数据错误,sql语句为{},异常如下:{}", sql, e);
        }
    }
}
 
Example #10
Source File: BinLogWriteEventHandler.java    From kkbinlog with Apache License 2.0 6 votes vote down vote up
@Override
protected EventBaseDTO formatData(Event event) {
    WriteRowsEventData d = event.getData();
    WriteRowsDTO writeRowsDTO = new WriteRowsDTO();
    writeRowsDTO.setEventType(DatabaseEvent.WRITE_ROWS);
    //添加表信息
    ColumnsTableMapEventData tableMapData = context.getTableMapData(d.getTableId());
    writeRowsDTO.setDatabase(tableMapData.getDatabase());
    writeRowsDTO.setTable(tableMapData.getTable());
    writeRowsDTO.setNamespace(context.getBinaryLogConfig().getNamespace());
    //添加列映射
    int[] includedColumns = d.getIncludedColumns().stream().toArray();
    writeRowsDTO.setRowMaps(d.getRows().stream()
            .map(r -> convert(r,includedColumns,tableMapData)).collect(Collectors.toList()));
    return writeRowsDTO;
}
 
Example #11
Source File: EventProcessor.java    From openmessaging-connect-odar with Apache License 2.0 6 votes vote down vote up
private void processXidEvent(Event event) {
    EventHeaderV4 header = event.getHeader();
    XidEventData data = event.getData();

    String binlogFilename = binaryLogClient.getBinlogFilename();
    Long position = header.getNextPosition();
    Long xid = data.getXid();

    BinlogPosition binlogPosition = new BinlogPosition(binlogFilename, position);
    transaction.setNextBinlogPosition(binlogPosition);
    transaction.setXid(xid);

    replicator.commit(transaction, true);

    transaction = new Transaction(config);
}
 
Example #12
Source File: ConsumerChannel.java    From syncer with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
FilterRes decide(SimpleEventType simpleEventType, BinlogDataId dataId, Event[] events) {
  if (logger.isTraceEnabled()) {
    logger.trace("Receive binlog event: {}", Arrays.toString(events));
  }
  SyncData[] aim = toSyncData(simpleEventType, dataId, events[0], events[1]);
  if (aim == null) { // not interested in this database+table
    return FilterRes.DENY;
  }

  boolean output = producerSink.output(aim);
  return output ? FilterRes.ACCEPT : FilterRes.DENY;
}
 
Example #13
Source File: EventProcessor.java    From openmessaging-connect-odar with Apache License 2.0 5 votes vote down vote up
private void processTableMapEvent(Event event) {
    TableMapEventData data = event.getData();
    String dbName = data.getDatabase();
    String tableName = data.getTable();
    Long tableId = data.getTableId();

    Table table = schema.getTable(dbName, tableName);
    if (table != null){
        tableMap.put(tableId, table);
    }
}
 
Example #14
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 5 votes vote down vote up
@Test
public void testIgnoredEvents() {
  eventHeader.setEventType(EventType.UNKNOWN);
  XAPrepareEventData eventData = new XAPrepareEventData();

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertFalse(binlogEvent.isPresent());
}
 
Example #15
Source File: ConsumerChannel.java    From syncer with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private SyncData[] toSyncData(SimpleEventType type, BinlogDataId dataId, Event... e) {
  TableMapEventData tableMap = e[0].getData();
  TableMeta table = consumerSchemaMeta.findTable(tableMap.getDatabase(), tableMap.getTable());
  if (table == null) {
    return null;
  }

  List<IndexedFullRow> indexedRow = RowsEvent
      .getIndexedRows(type, e[1].getData(), table.getPrimaryKeys());
  List<NamedFullRow> namedRow = RowsEvent
      .getNamedRows(indexedRow, table.getInterestedAndPkIndex(), table.getInterestedAndPkIndexToName());
  String primaryKey = RowsEvent.getPrimaryKey(table.getInterestedAndPkIndexToName(), table.getPrimaryKeys());
  SyncData[] res = new SyncData[namedRow.size()];
  boolean hasData = false;
  for (int i = 0; i < res.length; i++) {
    NamedFullRow row = namedRow.get(i);
    if (onlyUpdated && type == SimpleEventType.UPDATE && row.getUpdated().isEmpty()) {
      logger.debug("Discard {} because [{}]", dataId.eventId(), row);
      // even though in one update event, multiple rows can have different updated column,
      // so we can only skip one by one
      // e.g. we listening 'name1' but not 'name2' and the following update will make all updates in a single event
      // UPDATE relation
      // SET name1 = CASE WHEN userid1 = 3 THEN 'jack' ELSE name1 END,
      //     name2 = CASE WHEN userid2 = 3 THEN 'jack' ELSE name2 END
      continue;
    }
    Object pk = row.get(primaryKey);
    if (!table.isInterestedPK()) {
      row.remove(primaryKey);
    }
    hasData = true;
    res[i] = new SyncData(dataId.copyAndSetOrdinal(i), type, tableMap.getDatabase(), tableMap.getTable(), primaryKey, pk, row);
  }
  if (hasData) {
    return res;
  }
  return null;
}
 
Example #16
Source File: BinaryLogConnectorEventMapperTest.java    From SpinalTap with Apache License 2.0 5 votes vote down vote up
@Test
public void testFormatDescriptionEvent() {
  eventHeader.setEventType(EventType.FORMAT_DESCRIPTION);
  FormatDescriptionEventData eventData = new FormatDescriptionEventData();

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof StartEvent);
  StartEvent startEvent = (StartEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, startEvent.getBinlogFilePos());
  assertEquals(SERVER_ID, startEvent.getServerId());
  assertEquals(TIMESTAMP, startEvent.getTimestamp());
}
 
Example #17
Source File: MysqlDispatcher.java    From syncer with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public boolean dispatch(SimpleEventType simpleEventType, Object... data) {
  Preconditions.checkState(data.length == 2);
  Event[] events = new Event[]{(Event) data[0], (Event) data[1]};
  BinlogDataId dataId = DataId.fromEvent(events, binlogInfo.get().getBinlogFilename());
  MDC.put(LogbackLoggingField.EID, dataId.eventId());
  boolean res = true;
  for (ConsumerChannel consumerChannel : consumerChannels) {
    FilterRes decide = consumerChannel.decide(simpleEventType, dataId, events);
    res = res && FilterRes.ACCEPT == decide;
  }
  MDC.remove(LogbackLoggingField.EID);
  return res;
}
 
Example #18
Source File: DataId.java    From syncer with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * <a href="https://github.com/shyiko/mysql-binlog-connector-java/issues/200">binlog table
 * map</a>
 */
static BinlogDataId fromEvent(Event[] event, String binlogFileName) {
  EventHeaderV4 tableMap = event[0].getHeader();
  EventHeaderV4 second = event[1].getHeader();
  // have to remember table map event for restart
  // have to add data event position for unique id:
  // because one table map event may follow multiple data event
  return new BinlogDataId(binlogFileName, tableMap.getPosition(), second.getPosition());
}
 
Example #19
Source File: RawEventInvocationHandler.java    From replicator with Apache License 2.0 5 votes vote down vote up
public RawEventInvocationHandler(BinaryLogClient binaryLogClient, Event event) {
    this.gtidSet = binaryLogClient.getGtidSet();
    this.event = event;
    this.eventDataSubTypes = Stream
            .of(RawEventType.values())
            .map(RawEventType::getDefinition)
            .distinct()
            .collect(
                    Collectors.toMap(
                        (value) -> value.getSimpleName().replace("Raw", "").toLowerCase(),
                        (value) -> value
                    )
            );
}
 
Example #20
Source File: BinaryLogConsumer.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private SourceOffset createOffset(Event event) {
  if (isGtidEnabled()) {
    return new GtidSourceOffset(currentGtidSet)
        .withIncompleteTransaction(currentTxGtid, currentTxEventSeqNo);
  } else {
    return new BinLogPositionSourceOffset(
        currentBinLogFileName,
        ((EventHeaderV4) event.getHeader()).getNextPosition()
    );
  }
}
 
Example #21
Source File: BinaryLogConsumer.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private SourceOffset createOffset(Event event) {
  if (isGtidEnabled()) {
    return new GtidSourceOffset(currentGtidSet)
        .withIncompleteTransaction(currentTxGtid, currentTxEventSeqNo);
  } else {
    return new BinLogPositionSourceOffset(
        currentBinLogFileName,
        ((EventHeaderV4) event.getHeader()).getNextPosition()
    );
  }
}
 
Example #22
Source File: BinlogEventListener.java    From nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void onEvent(Event event) {
    while (!stopNow.get()) {
        RawBinlogEvent ep = new RawBinlogEvent(event, client.getBinlogFilename());
        try {
            if (queue.offer(ep, QUEUE_OFFER_TIMEOUT_MSEC, TimeUnit.MILLISECONDS)) {
                return;
            } else {
                throw new RuntimeException("Unable to add event to the queue");
            }
        } catch (InterruptedException e) {
            throw new RuntimeException("Interrupted while adding event to the queue");
        }
    }
}
 
Example #23
Source File: BinLogDDLEventHandler.java    From kkbinlog with Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
    boolean ddleventEnable = Boolean.parseBoolean(System.getProperty(DDLEVENT_ENABLE_KEY,DDLEVENT_ENABLE));
    if(ddleventEnable){
        QueryEventData data = event.getData();
        String sql = data.getSql();
        if (sql.contains(SUBSCRIBE_SQL_EVENT)) {
            log.info("数据库:{}发生alter table事件", data.getDatabase());
            String topic = System.getProperty(DEFAULT_TOPIC_KEY,DEFAULT_TOPIC);
            dataPublisher.pushToKafka(topic, data);
        }
    }
    updateBinaryLogStatus(event.getHeader());
}
 
Example #24
Source File: EventProcessor.java    From openmessaging-connect-odar with Apache License 2.0 5 votes vote down vote up
private void processWriteEvent(Event event) {
    WriteRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Serializable[]> list = data.getRows();

    for (Serializable[] row : list) {
        addRow(EntryType.CREATE, tableId, row, null);
    }
}
 
Example #25
Source File: EventProcessor.java    From openmessaging-connect-odar with Apache License 2.0 5 votes vote down vote up
private void processUpdateEvent(Event event) {
    UpdateRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Map.Entry<Serializable[], Serializable[]>> list = data.getRows();

    for (Map.Entry<Serializable[], Serializable[]> entry : list) {
        addRow(EntryType.UPDATE, tableId, entry.getValue(), entry.getKey());
    }
}
 
Example #26
Source File: EventProcessor.java    From openmessaging-connect-odar with Apache License 2.0 5 votes vote down vote up
private void processDeleteEvent(Event event) {
    DeleteRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Serializable[]> list = data.getRows();

    for (Serializable[] row : list) {
        addRow(EntryType.DELETE, tableId, null, row);
    }

}
 
Example #27
Source File: EventProcessor.java    From openmessaging-connect-odar with Apache License 2.0 5 votes vote down vote up
private void processQueryEvent(Event event) {
    QueryEventData data = event.getData();
    String sql = data.getSql();

    if (createTablePattern.matcher(sql).find()) {
        schema.reset();
    }
}
 
Example #28
Source File: BinLogWriteEventHandler.java    From kkbinlog with Apache License 2.0 5 votes vote down vote up
@Override
protected Set<ClientInfo> filter(Event event) {
    WriteRowsEventData d = event.getData();
    long tableId = d.getTableId();
    TableMapEventData tableMapEventData = context.getTableMapData(tableId);
    String tableKey = tableMapEventData.getDatabase().concat("/").concat(tableMapEventData.getTable());
    return clientInfoMap.get(tableKey);
}
 
Example #29
Source File: BinLogUpdateEventHandler.java    From kkbinlog with Apache License 2.0 5 votes vote down vote up
@Override
protected Set<ClientInfo> filter(Event event) {
    UpdateRowsEventData d = event.getData();
    long tableId = d.getTableId();
    TableMapEventData tableMapEventData =  context.getTableMapData(tableId);
    String tableKey = tableMapEventData.getDatabase().concat("/").concat(tableMapEventData.getTable());
    return clientInfoMap.get(tableKey);
}
 
Example #30
Source File: EventListener.java    From openmessaging-connect-odar with Apache License 2.0 5 votes vote down vote up
@Override
public void onEvent(Event event) {
    try {
        while (true) {
            if (queue.offer(event, 100, TimeUnit.MILLISECONDS)) {
                return;
            }
        }
    } catch (InterruptedException e) {
        log.error("Mysql task EventListener#onEvent error.", e);
    }
}