Java Code Examples for com.github.shyiko.mysql.binlog.event.Event

The following examples show how to use com.github.shyiko.mysql.binlog.event.Event. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 6 votes vote down vote up
private void processXidEvent(Event event) {
    EventHeaderV4 header = event.getHeader();
    XidEventData data = event.getData();

    String binlogFilename = binaryLogClient.getBinlogFilename();
    Long position = header.getNextPosition();
    Long xid = data.getXid();

    BinlogPosition binlogPosition = new BinlogPosition(binlogFilename, position);
    transaction.setNextBinlogPosition(binlogPosition);
    transaction.setXid(xid);

    replicator.commit(transaction, true);

    transaction = new Transaction(config);
}
 
Example 2
Source Project: kkbinlog   Source File: BinLogWriteEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected EventBaseDTO formatData(Event event) {
    WriteRowsEventData d = event.getData();
    WriteRowsDTO writeRowsDTO = new WriteRowsDTO();
    writeRowsDTO.setEventType(DatabaseEvent.WRITE_ROWS);
    //添加表信息
    ColumnsTableMapEventData tableMapData = context.getTableMapData(d.getTableId());
    writeRowsDTO.setDatabase(tableMapData.getDatabase());
    writeRowsDTO.setTable(tableMapData.getTable());
    writeRowsDTO.setNamespace(context.getBinaryLogConfig().getNamespace());
    //添加列映射
    int[] includedColumns = d.getIncludedColumns().stream().toArray();
    writeRowsDTO.setRowMaps(d.getRows().stream()
            .map(r -> convert(r,includedColumns,tableMapData)).collect(Collectors.toList()));
    return writeRowsDTO;
}
 
Example 3
Source Project: kkbinlog   Source File: BinLogUpdateEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected EventBaseDTO formatData(Event event) {
    UpdateRowsEventData d = event.getData();
    UpdateRowsDTO updateRowsDTO = new UpdateRowsDTO();
    updateRowsDTO.setEventType(DatabaseEvent.UPDATE_ROWS);
    //添加表信息
    ColumnsTableMapEventData tableMapData = context.getTableMapData(d.getTableId());
    updateRowsDTO.setDatabase(tableMapData.getDatabase());
    updateRowsDTO.setTable(tableMapData.getTable());
    updateRowsDTO.setNamespace(context.getBinaryLogConfig().getNamespace());
    //添加列映射
    int[] includedColumns = d.getIncludedColumns().stream().toArray();
    List<UpdateRow> urs = d.getRows().stream()
            .map(e -> new UpdateRow(convert(e.getKey(),includedColumns,tableMapData),
                    convert(e.getValue(),includedColumns,tableMapData))).collect(Collectors.toList());
    updateRowsDTO.setRows(urs);
    return updateRowsDTO;
}
 
Example 4
Source Project: kkbinlog   Source File: BinLogTableMapEventHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void handle(Event event) {
    TableMapEventData d = event.getData();
    log.debug("TableMapEventData:{}", d);
    ColumnsTableMapEventData tableMapEventData = context.getTableMapData(d.getTableId());
    //如果表结构有变化,重新设置映射信息
    if (tableMapEventData == null || !ColumnsTableMapEventData.checkEqual(d, tableMapEventData)) {
        log.info("更新表映射:{}-{}", d.getDatabase(), d.getTable());
        ColumnsTableMapEventData data = new ColumnsTableMapEventData(d);
        String sql = "show columns from `" + d.getTable() + "` from `" + d.getDatabase() + "`";
        try (Connection conn = context.getDataSource().getConnection();
             PreparedStatement ps = conn.prepareStatement(sql);
             ResultSet resultSet = ps.executeQuery();) {
            while (resultSet.next()) {
                data.addColumnName(resultSet.getString("Field"));
            }
            //将表id和表映射
            context.addTableMapData(d.getTableId(), data);
        } catch (SQLException e) {
            log.error("获取表数据错误,sql语句为{},异常如下:{}", sql, e);
        }
    }
}
 
Example 5
Source Project: SpinalTap   Source File: BinaryLogConnectorSource.java    License: Apache License 2.0 6 votes vote down vote up
public void onEvent(Event event) {
  Preconditions.checkState(isStarted(), "Source is not started and should not process events");

  final EventHeaderV4 header = event.getHeader();
  final BinlogFilePos filePos =
      new BinlogFilePos(
          binlogClient.getBinlogFilename(),
          header.getPosition(),
          header.getNextPosition(),
          binlogClient.getGtidSet(),
          serverUUID);

  BinaryLogConnectorEventMapper.INSTANCE
      .map(event, filePos)
      .ifPresent(BinaryLogConnectorSource.super::processEvent);
}
 
Example 6
@Test
public void testWriteEvent() {
  eventHeader.setEventType(EventType.EXT_WRITE_ROWS);
  WriteRowsEventData eventData = new WriteRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(ROW));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof WriteEvent);
  WriteEvent writeEvent = (WriteEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, writeEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(ROW), writeEvent.getRows());
  assertEquals(SERVER_ID, writeEvent.getServerId());
  assertEquals(TABLE_ID, writeEvent.getTableId());
  assertEquals(TIMESTAMP, writeEvent.getTimestamp());
}
 
Example 7
@Test
public void testUpdateEvent() {
  eventHeader.setEventType(EventType.EXT_UPDATE_ROWS);
  UpdateRowsEventData eventData = new UpdateRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(Maps.immutableEntry(PREV_ROW, ROW)));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof UpdateEvent);
  UpdateEvent updateEvent = (UpdateEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, updateEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(Maps.immutableEntry(PREV_ROW, ROW)), updateEvent.getRows());
  assertEquals(SERVER_ID, updateEvent.getServerId());
  assertEquals(TABLE_ID, updateEvent.getTableId());
  assertEquals(TIMESTAMP, updateEvent.getTimestamp());
}
 
Example 8
@Test
public void testDeleteEvent() {
  eventHeader.setEventType(EventType.EXT_DELETE_ROWS);
  DeleteRowsEventData eventData = new DeleteRowsEventData();
  eventData.setTableId(TABLE_ID);
  eventData.setRows(ImmutableList.of(PREV_ROW));

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof DeleteEvent);
  DeleteEvent deleteEvent = (DeleteEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, deleteEvent.getBinlogFilePos());
  assertEquals(ImmutableList.of(PREV_ROW), deleteEvent.getRows());
  assertEquals(SERVER_ID, deleteEvent.getServerId());
  assertEquals(TABLE_ID, deleteEvent.getTableId());
  assertEquals(TIMESTAMP, deleteEvent.getTimestamp());
}
 
Example 9
@Test
public void testTableMapEvent() {
  eventHeader.setEventType(EventType.TABLE_MAP);
  TableMapEventData eventData = new TableMapEventData();
  eventData.setDatabase(DATABASE);
  eventData.setTable(TABLE);
  eventData.setTableId(TABLE_ID);
  eventData.setColumnTypes(new byte[] {(byte) 0, (byte) 1, (byte) 2});

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof TableMapEvent);
  TableMapEvent tableMapEvent = (TableMapEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, tableMapEvent.getBinlogFilePos());
  assertEquals(DATABASE, tableMapEvent.getDatabase());
  assertEquals(TABLE, tableMapEvent.getTable());
  assertEquals(TABLE_ID, tableMapEvent.getTableId());
  assertEquals(
      ImmutableList.of(ColumnDataType.DECIMAL, ColumnDataType.TINY, ColumnDataType.SHORT),
      tableMapEvent.getColumnTypes());
}
 
Example 10
@Test
public void testXidEvent() {
  long xid = 88888L;
  eventHeader.setEventType(EventType.XID);
  XidEventData eventData = new XidEventData();
  eventData.setXid(xid);

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof XidEvent);
  XidEvent xidEvent = (XidEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, xidEvent.getBinlogFilePos());
  assertEquals(SERVER_ID, xidEvent.getServerId());
  assertEquals(TIMESTAMP, xidEvent.getTimestamp());
  assertEquals(xid, xidEvent.getXid());
}
 
Example 11
@Test
public void testQueryEvent() {
  String sql = "CREATE UNIQUE INDEX unique_index ON `my_db`.`my_table` (`col1`, `col2`)";
  eventHeader.setEventType(EventType.QUERY);
  QueryEventData eventData = new QueryEventData();
  eventData.setDatabase(DATABASE);
  eventData.setSql(sql);

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof QueryEvent);
  QueryEvent queryEvent = (QueryEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, queryEvent.getBinlogFilePos());
  assertEquals(DATABASE, queryEvent.getDatabase());
  assertEquals(SERVER_ID, queryEvent.getServerId());
  assertEquals(TIMESTAMP, queryEvent.getTimestamp());
  assertEquals(sql, queryEvent.getSql());
}
 
Example 12
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 5 votes vote down vote up
private void processTableMapEvent(Event event) {
    TableMapEventData data = event.getData();
    String dbName = data.getDatabase();
    String tableName = data.getTable();
    Long tableId = data.getTableId();

    Table table = schema.getTable(dbName, tableName);
    if (table != null){
        tableMap.put(tableId, table);
    }
}
 
Example 13
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 5 votes vote down vote up
private void processWriteEvent(Event event) {
    WriteRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Serializable[]> list = data.getRows();

    for (Serializable[] row : list) {
        addRow(EntryType.CREATE, tableId, row, null);
    }
}
 
Example 14
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 5 votes vote down vote up
private void processUpdateEvent(Event event) {
    UpdateRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Map.Entry<Serializable[], Serializable[]>> list = data.getRows();

    for (Map.Entry<Serializable[], Serializable[]> entry : list) {
        addRow(EntryType.UPDATE, tableId, entry.getValue(), entry.getKey());
    }
}
 
Example 15
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 5 votes vote down vote up
private void processDeleteEvent(Event event) {
    DeleteRowsEventData data = event.getData();
    Long tableId = data.getTableId();
    List<Serializable[]> list = data.getRows();

    for (Serializable[] row : list) {
        addRow(EntryType.DELETE, tableId, null, row);
    }

}
 
Example 16
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 5 votes vote down vote up
private void processQueryEvent(Event event) {
    QueryEventData data = event.getData();
    String sql = data.getSql();

    if (createTablePattern.matcher(sql).find()) {
        schema.reset();
    }
}
 
Example 17
Source Project: openmessaging-connect-odar   Source File: EventListener.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onEvent(Event event) {
    try {
        while (true) {
            if (queue.offer(event, 100, TimeUnit.MILLISECONDS)) {
                return;
            }
        }
    } catch (InterruptedException e) {
        log.error("Mysql task EventListener#onEvent error.", e);
    }
}
 
Example 18
Source Project: kkbinlog   Source File: BinLogWriteEventHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Set<ClientInfo> filter(Event event) {
    WriteRowsEventData d = event.getData();
    long tableId = d.getTableId();
    TableMapEventData tableMapEventData = context.getTableMapData(tableId);
    String tableKey = tableMapEventData.getDatabase().concat("/").concat(tableMapEventData.getTable());
    return clientInfoMap.get(tableKey);
}
 
Example 19
Source Project: kkbinlog   Source File: BinLogUpdateEventHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Set<ClientInfo> filter(Event event) {
    UpdateRowsEventData d = event.getData();
    long tableId = d.getTableId();
    TableMapEventData tableMapEventData =  context.getTableMapData(tableId);
    String tableKey = tableMapEventData.getDatabase().concat("/").concat(tableMapEventData.getTable());
    return clientInfoMap.get(tableKey);
}
 
Example 20
Source Project: kkbinlog   Source File: BinLogDDLEventHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handle(Event event) {
    boolean ddleventEnable = Boolean.parseBoolean(System.getProperty(DDLEVENT_ENABLE_KEY,DDLEVENT_ENABLE));
    if(ddleventEnable){
        QueryEventData data = event.getData();
        String sql = data.getSql();
        if (sql.contains(SUBSCRIBE_SQL_EVENT)) {
            log.info("数据库:{}发生alter table事件", data.getDatabase());
            String topic = System.getProperty(DEFAULT_TOPIC_KEY,DEFAULT_TOPIC);
            dataPublisher.pushToKafka(topic, data);
        }
    }
    updateBinaryLogStatus(event.getHeader());
}
 
Example 21
@Test
public void testFormatDescriptionEvent() {
  eventHeader.setEventType(EventType.FORMAT_DESCRIPTION);
  FormatDescriptionEventData eventData = new FormatDescriptionEventData();

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertTrue(binlogEvent.isPresent());
  assertTrue(binlogEvent.get() instanceof StartEvent);
  StartEvent startEvent = (StartEvent) (binlogEvent.get());
  assertEquals(BINLOG_FILE_POS, startEvent.getBinlogFilePos());
  assertEquals(SERVER_ID, startEvent.getServerId());
  assertEquals(TIMESTAMP, startEvent.getTimestamp());
}
 
Example 22
@Test
public void testIgnoredEvents() {
  eventHeader.setEventType(EventType.UNKNOWN);
  XAPrepareEventData eventData = new XAPrepareEventData();

  Optional<BinlogEvent> binlogEvent =
      BinaryLogConnectorEventMapper.INSTANCE.map(
          new Event(eventHeader, eventData), BINLOG_FILE_POS);
  assertFalse(binlogEvent.isPresent());
}
 
Example 23
FilterRes decide(SimpleEventType simpleEventType, BinlogDataId dataId, Event[] events) {
  if (logger.isTraceEnabled()) {
    logger.trace("Receive binlog event: {}", Arrays.toString(events));
  }
  SyncData[] aim = toSyncData(simpleEventType, dataId, events[0], events[1]);
  if (aim == null) { // not interested in this database+table
    return FilterRes.DENY;
  }

  boolean output = producerSink.output(aim);
  return output ? FilterRes.ACCEPT : FilterRes.DENY;
}
 
Example 24
private SyncData[] toSyncData(SimpleEventType type, BinlogDataId dataId, Event... e) {
  TableMapEventData tableMap = e[0].getData();
  TableMeta table = consumerSchemaMeta.findTable(tableMap.getDatabase(), tableMap.getTable());
  if (table == null) {
    return null;
  }

  List<IndexedFullRow> indexedRow = RowsEvent
      .getIndexedRows(type, e[1].getData(), table.getPrimaryKeys());
  List<NamedFullRow> namedRow = RowsEvent
      .getNamedRows(indexedRow, table.getInterestedAndPkIndex(), table.getInterestedAndPkIndexToName());
  String primaryKey = RowsEvent.getPrimaryKey(table.getInterestedAndPkIndexToName(), table.getPrimaryKeys());
  SyncData[] res = new SyncData[namedRow.size()];
  boolean hasData = false;
  for (int i = 0; i < res.length; i++) {
    NamedFullRow row = namedRow.get(i);
    if (onlyUpdated && type == SimpleEventType.UPDATE && row.getUpdated().isEmpty()) {
      logger.debug("Discard {} because [{}]", dataId.eventId(), row);
      // even though in one update event, multiple rows can have different updated column,
      // so we can only skip one by one
      // e.g. we listening 'name1' but not 'name2' and the following update will make all updates in a single event
      // UPDATE relation
      // SET name1 = CASE WHEN userid1 = 3 THEN 'jack' ELSE name1 END,
      //     name2 = CASE WHEN userid2 = 3 THEN 'jack' ELSE name2 END
      continue;
    }
    Object pk = row.get(primaryKey);
    if (!table.isInterestedPK()) {
      row.remove(primaryKey);
    }
    hasData = true;
    res[i] = new SyncData(dataId.copyAndSetOrdinal(i), type, tableMap.getDatabase(), tableMap.getTable(), primaryKey, pk, row);
  }
  if (hasData) {
    return res;
  }
  return null;
}
 
Example 25
@Override
public boolean dispatch(SimpleEventType simpleEventType, Object... data) {
  Preconditions.checkState(data.length == 2);
  Event[] events = new Event[]{(Event) data[0], (Event) data[1]};
  BinlogDataId dataId = DataId.fromEvent(events, binlogInfo.get().getBinlogFilename());
  MDC.put(LogbackLoggingField.EID, dataId.eventId());
  boolean res = true;
  for (ConsumerChannel consumerChannel : consumerChannels) {
    FilterRes decide = consumerChannel.decide(simpleEventType, dataId, events);
    res = res && FilterRes.ACCEPT == decide;
  }
  MDC.remove(LogbackLoggingField.EID);
  return res;
}
 
Example 26
Source Project: syncer   Source File: DataId.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * <a href="https://github.com/shyiko/mysql-binlog-connector-java/issues/200">binlog table
 * map</a>
 */
static BinlogDataId fromEvent(Event[] event, String binlogFileName) {
  EventHeaderV4 tableMap = event[0].getHeader();
  EventHeaderV4 second = event[1].getHeader();
  // have to remember table map event for restart
  // have to add data event position for unique id:
  // because one table map event may follow multiple data event
  return new BinlogDataId(binlogFileName, tableMap.getPosition(), second.getPosition());
}
 
Example 27
Source Project: replicator   Source File: RawEventInvocationHandler.java    License: Apache License 2.0 5 votes vote down vote up
public RawEventInvocationHandler(BinaryLogClient binaryLogClient, Event event) {
    this.gtidSet = binaryLogClient.getGtidSet();
    this.event = event;
    this.eventDataSubTypes = Stream
            .of(RawEventType.values())
            .map(RawEventType::getDefinition)
            .distinct()
            .collect(
                    Collectors.toMap(
                        (value) -> value.getSimpleName().replace("Raw", "").toLowerCase(),
                        (value) -> value
                    )
            );
}
 
Example 28
Source Project: datacollector   Source File: BinaryLogConsumer.java    License: Apache License 2.0 5 votes vote down vote up
private SourceOffset createOffset(Event event) {
  if (isGtidEnabled()) {
    return new GtidSourceOffset(currentGtidSet)
        .withIncompleteTransaction(currentTxGtid, currentTxEventSeqNo);
  } else {
    return new BinLogPositionSourceOffset(
        currentBinLogFileName,
        ((EventHeaderV4) event.getHeader()).getNextPosition()
    );
  }
}
 
Example 29
Source Project: datacollector   Source File: BinaryLogConsumer.java    License: Apache License 2.0 5 votes vote down vote up
private SourceOffset createOffset(Event event) {
  if (isGtidEnabled()) {
    return new GtidSourceOffset(currentGtidSet)
        .withIncompleteTransaction(currentTxGtid, currentTxEventSeqNo);
  } else {
    return new BinLogPositionSourceOffset(
        currentBinLogFileName,
        ((EventHeaderV4) event.getHeader()).getNextPosition()
    );
  }
}
 
Example 30
Source Project: nifi   Source File: BinlogEventListener.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onEvent(Event event) {
    while (!stopNow.get()) {
        RawBinlogEvent ep = new RawBinlogEvent(event, client.getBinlogFilename());
        try {
            if (queue.offer(ep, QUEUE_OFFER_TIMEOUT_MSEC, TimeUnit.MILLISECONDS)) {
                return;
            } else {
                throw new RuntimeException("Unable to add event to the queue");
            }
        } catch (InterruptedException e) {
            throw new RuntimeException("Interrupted while adding event to the queue");
        }
    }
}