Java Code Examples for com.github.shyiko.mysql.binlog.BinaryLogClient

The following examples show how to use com.github.shyiko.mysql.binlog.BinaryLogClient. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kkbinlog   Source File: BinLogClientFactory.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 配置当前binlog位置
 * @param client
 * @param binaryLogConfig
 */
private void configBinaryLogStatus(BinaryLogClient client, BinaryLogConfig binaryLogConfig) {

    JSONObject binLogStatus = etcdService.getBinaryLogStatus(binaryLogConfig);

    if (binLogStatus != null) {
        Object binlogFilename = binLogStatus.get("binlogFilename");
        if (binlogFilename != null) {
            client.setBinlogFilename((String) binlogFilename);
        }
        Long binlogPosition = binLogStatus.getLong("binlogPosition");
        if (binlogPosition != null) {
            client.setBinlogPosition(binlogPosition);
        }
    }
}
 
Example 2
Source Project: easy-sync   Source File: MysqlSyncListener.java    License: Apache License 2.0 6 votes vote down vote up
public void init(TimeTaskContext context, Producer producer, BinaryLogClient client, DatabaseService databaseService, PositionSaveService positionSaveService) {
    this.context = context;
    this.producer = producer;
    this.client = client;
    this.databaseService = databaseService;
    this.positionSaveService = positionSaveService;
    this.timezoneInMS = databaseService.getTimezoneHours() * 3600 * 1000;

   /* position=positionService.getPosition(databaseService.getTimeTaskId(),databaseService.getServerId());
    if(position==null) {
        position = EntityUtils.create(Position.class);
        position.setTimeTaskId(databaseService.getTimeTaskId());
        position.setServerId(databaseService.getServerId());
    }*/


}
 
Example 3
Source Project: SpinalTap   Source File: BinaryLogConnectorSource.java    License: Apache License 2.0 6 votes vote down vote up
public void onCommunicationFailure(BinaryLogClient client, Exception ex) {
  log.error(
      String.format(
          "Communication failure from source %s, binlogFile=%s, binlogPos=%s",
          name, client.getBinlogFilename(), client.getBinlogPosition()),
      ex);

  if (ex.getMessage().startsWith(INVALID_BINLOG_POSITION_ERROR_CODE)) {
    ex =
        new InvalidBinlogPositionException(
            String.format(
                "Invalid position %s in binlog file %s",
                client.getBinlogPosition(), client.getBinlogFilename()));
  }

  onCommunicationError(ex);
}
 
Example 4
Source Project: replicator   Source File: BinaryLogSupplier.java    License: Apache License 2.0 6 votes vote down vote up
private BinaryLogClient getClient(String hostname) {

        // TODO: Implement status variable parser: https://github.com/shyiko/mysql-binlog-connector-java/issues/174
        BinaryLogClient client = new BinaryLogClient(
                hostname,
                this.port,
                this.schema,
                this.username,
                this.password
        );
        client.setHeartbeatInterval(TimeUnit.MILLISECONDS.toMillis(1000));

        EventDeserializer eventDeserializer = new EventDeserializer();
        eventDeserializer.setCompatibilityMode(
                EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY,
                EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG);
        client.setEventDeserializer(eventDeserializer);

        return client;
    }
 
Example 5
Source Project: datacollector   Source File: MysqlBinLogSource.java    License: Apache License 2.0 6 votes vote down vote up
private BinaryLogClient createBinaryLogClient(MysqlBinLogSourceConfig config) {
  BinaryLogClient binLogClient = new BinaryLogClient(
      sshTunnel.getHost(),
      sshTunnel.getPort(),
      config.username.get(),
      config.password.get()
  );

  switch (config.sslMode) {
    case REQUIRED:
      binLogClient.setSSLMode(SSLMode.REQUIRED);
      break;
    case VERIFY_CA:
      binLogClient.setSSLMode(SSLMode.VERIFY_CA);
      setSSLSocketFactory(binLogClient);
      break;
    case VERIFY_IDENTITY:
      binLogClient.setSSLMode(SSLMode.VERIFY_IDENTITY);
      setSSLSocketFactory(binLogClient);
      break;
  }

  binLogClient.setServerId(config.serverId);
  return binLogClient;
}
 
Example 6
Source Project: ad   Source File: BinlogClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 该方法应该在应用程序启动时刻 就开始进行binlog 监听,
 * 可以使用springboot 提供的{@link org.springframework.boot.CommandLineRunner}
 * 或者{@link org.springframework.boot.ApplicationRunner}默认优先级更高一点
 * @see org.springframework.boot.SpringApplication#callRunners(ApplicationContext, ApplicationArguments)
 *
 * 此处实现了在 {@link top.ezttf.ad.runner.BinlogRunner} 中直接调用
 */
public void connect() {
    int corePoolSize = 1, maximumPoolSize = 1;
    long keepAliveTime = 0L;
    BlockingQueue<Runnable> workQueue = new LinkedBlockingQueue<>(1);
    ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("binlog-thread-%d").build();
    ThreadPoolExecutor executor = new ThreadPoolExecutor(corePoolSize,
            maximumPoolSize,
            keepAliveTime,
            TimeUnit.MILLISECONDS,
            workQueue,
            threadFactory,
            new ThreadPoolExecutor.AbortPolicy()
    );
    executor.submit(() -> {
        client = new BinaryLogClient(
                config.getHost(),
                config.getPort(),
                config.getUsername(),
                config.getPassword()
        );
        if (StringUtils.isNotBlank(config.getBinlogName())
                && !config.getPosition().equals(-1L)) {
            client.setBinlogFilename(config.getBinlogName());
            client.setBinlogPosition(config.getPosition());
        }
        client.registerEventListener(listener);
        try {
            log.info("connecting to mysql...");
            client.connect();
            log.info("connected to mysql");
        } catch (Exception e) {
            e.printStackTrace();
            log.error("mysql binlog connect error");
        }
    });
    executor.shutdown();
}
 
Example 7
Source Project: SpinalTap   Source File: BinaryLogConnectorSource.java    License: Apache License 2.0 5 votes vote down vote up
public BinaryLogConnectorSource(
    @NonNull final String name,
    @NonNull final MysqlConfiguration config,
    final TlsConfiguration tlsConfig,
    @NonNull final BinaryLogClient binlogClient,
    @NonNull final MysqlClient mysqlClient,
    @NonNull final TableCache tableCache,
    @NonNull final StateRepository stateRepository,
    @NonNull final StateHistory stateHistory,
    @NonNull final MysqlSchemaManager schemaManager,
    @NonNull final MysqlSourceMetrics metrics,
    @NonNull final AtomicLong currentLeaderEpoch) {
  super(
      name,
      new DataSource(config.getHost(), config.getPort(), name),
      new HashSet<>(config.getCanonicalTableNames()),
      tableCache,
      stateRepository,
      stateHistory,
      config.getInitialBinlogFilePosition(),
      schemaManager,
      metrics,
      currentLeaderEpoch,
      new AtomicReference<>(),
      new AtomicReference<>());

  this.binlogClient = binlogClient;
  this.mysqlClient = mysqlClient;
  this.serverUUID = mysqlClient.getServerUUID();
  initializeClient(config, tlsConfig);
}
 
Example 8
Source Project: SpinalTap   Source File: BinaryLogConnectorSource.java    License: Apache License 2.0 5 votes vote down vote up
public void onEventDeserializationFailure(BinaryLogClient client, Exception ex) {
  log.error(
      String.format(
          "Deserialization failure from source %s, BinlogFile=%s, binlogPos=%s",
          name, client.getBinlogFilename(), client.getBinlogPosition()),
      ex);

  onDeserializationError(ex);
}
 
Example 9
Source Project: SpinalTap   Source File: BinaryLogConnectorSource.java    License: Apache License 2.0 5 votes vote down vote up
public void onDisconnect(BinaryLogClient client) {
  log.info(
      "Disconnected from source {}. BinlogFile={}, binlogPos={}",
      name,
      client.getBinlogFilename(),
      client.getBinlogPosition());
  metrics.clientDisconnected();
  started.set(false);
}
 
Example 10
private void configLogClient(MysqlConnection connection, String password,
                             ConsumerRegistry registry) throws IOException {
  client = new BinaryLogClient(connection.getAddress(), connection.getPort(),
      connection.getUser(), password);
  client.registerLifecycleListener(new LogLifecycleListener());
  client.setEventDeserializer(SyncDeserializer.defaultDeserializer());
  client.setServerId(random.nextInt(Integer.MAX_VALUE));
  client.setSSLMode(SSLMode.DISABLED);
  BinlogInfo binlogInfo = registry.votedBinlogInfo(connection);
  logger.info("Producer[{}] connect to {}", connectorIdentifier, binlogInfo);
  client.setBinlogFilename(binlogInfo.getBinlogFilename());
  client.setBinlogPosition(binlogInfo.getBinlogPosition());
}
 
Example 11
@Override
public void onCommunicationFailure(BinaryLogClient client, Exception ex) {
  if (binlogDeprecated(ex)) {
    if (dupServerId(ex)) {
      throw new DupServerIdException(ex);
    }
    throw new InvalidBinlogException(ex, client.getBinlogFilename(), client.getBinlogPosition());
  } else {
    logger.error("Communication failure", ex);
  }
}
 
Example 12
Source Project: Mycat2   Source File: BinlogStream.java    License: GNU General Public License v3.0 5 votes vote down vote up
private synchronized BinaryLogClient allocateBinaryLogClient() {
    if (isConnected()) {
        throw new IllegalStateException("MySQL replication stream is already open");
    }
    binaryLogClient = new BinaryLogClient(hostname, port, username, password);
    binaryLogClient.setBinlogFilename(getBinglogFile());
    binaryLogClient.setBinlogPosition(getBinlogPos());
    binaryLogClient.setServerId(getSlaveID());
    binaryLogClient.registerEventListener(new DelegatingEventListener());
    return binaryLogClient;
}
 
Example 13
Source Project: replicator   Source File: RawEventInvocationHandler.java    License: Apache License 2.0 5 votes vote down vote up
public RawEventInvocationHandler(BinaryLogClient binaryLogClient, Event event) {
    this.gtidSet = binaryLogClient.getGtidSet();
    this.event = event;
    this.eventDataSubTypes = Stream
            .of(RawEventType.values())
            .map(RawEventType::getDefinition)
            .distinct()
            .collect(
                    Collectors.toMap(
                        (value) -> value.getSimpleName().replace("Raw", "").toLowerCase(),
                        (value) -> value
                    )
            );
}
 
Example 14
Source Project: datacollector   Source File: MysqlSource.java    License: Apache License 2.0 5 votes vote down vote up
private BinaryLogClient createBinaryLogClient() {
  BinaryLogClient binLogClient = new BinaryLogClient(
      getConfig().hostname,
      port,
      getConfig().username.get(),
      getConfig().password.get()
  );
  if (getConfig().useSsl) {
    binLogClient.setSSLMode(SSLMode.REQUIRED);
  } else {
    binLogClient.setSSLMode(SSLMode.DISABLED);
  }
  binLogClient.setServerId(serverId);
  return binLogClient;
}
 
Example 15
Source Project: ad   Source File: BinlogServiceTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBinlog() throws IOException {
    String hostname = "127.0.0.1", username = "yuwen", password = "lyp82nlf";
    int port = 3306;
    // BinaryLogClient其实就是一个连接数据库的客户端,
    // 它加自己伪装成slave 连接到master上
    BinaryLogClient client = new BinaryLogClient(
            hostname, port, username, password
    );
    // 设置监听的Binlog, 如果不设置则监听最新的Binlog
    //client.setBinlogFilename();
    // 设置监听的binlog位置, 如果不设置则监听最新的位置
    //client.setBinlogPosition();
    // 注册事件监听器, 监听期间MySQL发生的一些变化, Event代表已经发生的事件
    client.registerEventListener(event -> {
        // MySQL 数据表发生变化的一些数据
        EventData eventData = event.getData();
        if (eventData instanceof UpdateRowsEventData) {
            log.info("update event");
            log.debug("{}", eventData);
        } else if (eventData instanceof WriteRowsEventData) {
            log.info("write event");
            log.debug("{}", eventData);
        } else if (eventData instanceof DeleteRowsEventData) {
            log.info("delete event");
            log.debug("{}", eventData);
        }
    });
    // 连接到 master 开始监听
    client.connect();


    // 启动后手动连接到 MySQL执行
    // insert into `ad_unit_keyword` (`unit_id`, `keyword`) values (10, '标志');
    // 控制台得到日志
    // 15:39:17.410 [main] INFO top.ezttf.ad.service.BinlogServiceTest - write event
    // 15:39:17.459 [main] DEBUG top.ezttf.ad.service.BinlogServiceTest - WriteRowsEventData{tableId=122, includedColumns={0, 1, 2}, rows=[
    //     [13, 10, 标志]
    // ]}

    // WriteRowsEventData{tableId=118, includedColumns={0, 1, 2, 3, 4, 5, 6, 7}, rows=[
    //    [11, 666, plan, 1, 2019-01-01, 2019-01-01, Tue Jan 01 08:00:00 CST 2019, Tue Jan 01 08:00:00 CST 2019]
    //]}
}
 
Example 16
Source Project: openmessaging-connect-odar   Source File: EventProcessor.java    License: Apache License 2.0 4 votes vote down vote up
public void start() throws Exception {

        initDataSource();

        binlogPositionManager = new BinlogPositionManager(config, dataSource);
        binlogPositionManager.initBeginPosition();

        schema = new Schema(dataSource);
        String whiteDataBases = config.getWhiteDataBase();
        String whiteTables = config.getWhiteTable();

        if (!StringUtils.isEmpty(whiteDataBases)){
            Arrays.asList(whiteDataBases.trim().split(",")).forEach(whiteDataBase ->{
                Collections.addAll(schema.dataBaseWhiteList, whiteDataBase);
            });
        }

        if (!StringUtils.isEmpty(whiteTables)){
            Arrays.asList(whiteTables.trim().split(",")).forEach(whiteTable ->{
                Collections.addAll(schema.tableWhiteList, whiteTable);
            });
        }
        schema.load();

        eventListener = new EventListener(queue);
        binaryLogClient = new BinaryLogClient(config.mysqlAddr,
            config.mysqlPort,
            config.mysqlUsername,
            config.mysqlPassword);
        binaryLogClient.setBlocking(true);
        binaryLogClient.setServerId(1001);

        EventDeserializer eventDeserializer = new EventDeserializer();
        eventDeserializer.setCompatibilityMode(EventDeserializer.CompatibilityMode.DATE_AND_TIME_AS_LONG,
            EventDeserializer.CompatibilityMode.CHAR_AND_BINARY_AS_BYTE_ARRAY);
        binaryLogClient.setEventDeserializer(eventDeserializer);
        binaryLogClient.registerEventListener(eventListener);
        binaryLogClient.setBinlogFilename(binlogPositionManager.getBinlogFilename());
        binaryLogClient.setBinlogPosition(binlogPositionManager.getPosition());

        binaryLogClient.connect(3000);

        LOGGER.info("Started.");

        stopped = false;

        doProcess();
    }
 
Example 17
Source Project: SpinalTap   Source File: BinaryLogConnectorSource.java    License: Apache License 2.0 4 votes vote down vote up
public void onConnect(BinaryLogClient client) {
  log.info("Connected to source {}.", name);
  metrics.clientConnected();
}
 
Example 18
Source Project: SpinalTap   Source File: MysqlSourceFactory.java    License: Apache License 2.0 4 votes vote down vote up
public Source create(
    @NonNull final MysqlConfiguration configuration,
    @NonNull final String user,
    @NonNull final String password,
    @Min(0) final long serverId,
    final TlsConfiguration tlsConfiguration,
    @NonNull final Repository<SourceState> backingStateRepository,
    @NonNull final Repository<Collection<SourceState>> stateHistoryRepository,
    final MysqlSchemaManagerFactory schemaManagerFactory,
    @NonNull final MysqlSourceMetrics metrics,
    @Min(0) final long leaderEpoch) {
  final String name = configuration.getName();
  final String host = configuration.getHost();
  final int port = configuration.getPort();

  final BinaryLogClient binlogClient = new BinaryLogClient(host, port, user, password);

  /* Override the global server_id if it is set in MysqlConfiguration
    Allow each source to use a different server_id
  */
  if (configuration.getServerId() != MysqlConfiguration.DEFAULT_SERVER_ID) {
    binlogClient.setServerId(configuration.getServerId());
  } else {
    binlogClient.setServerId(serverId);
  }

  final StateRepository stateRepository =
      new StateRepository(name, backingStateRepository, metrics);
  final StateHistory stateHistory = new StateHistory(name, stateHistoryRepository, metrics);

  final MysqlClient mysqlClient =
      MysqlClient.create(
          host, port, user, password, configuration.isMTlsEnabled(), tlsConfiguration);

  final MysqlSchemaManager schemaManager =
      schemaManagerFactory.create(
          name, mysqlClient, configuration.isSchemaVersionEnabled(), metrics);

  final TableCache tableCache =
      new TableCache(schemaManager, configuration.getOverridingDatabase());

  final BinaryLogConnectorSource source =
      new BinaryLogConnectorSource(
          name,
          configuration,
          tlsConfiguration,
          binlogClient,
          mysqlClient,
          tableCache,
          stateRepository,
          stateHistory,
          schemaManager,
          metrics,
          new AtomicLong(leaderEpoch));

  source.addEventValidator(new EventOrderValidator(metrics::outOfOrder));
  source.addMutationValidator(new MutationOrderValidator(metrics::outOfOrder));
  source.addMutationValidator(new MutationSchemaValidator(metrics::invalidSchema));

  return source;
}
 
Example 19
@Override
public void onConnect(BinaryLogClient client) {
  logger.info("Connected {}@{}", client.getBinlogFilename(), client.getBinlogPosition());
}
 
Example 20
@Override
public void onEventDeserializationFailure(BinaryLogClient client, Exception ex) {
  logger.error("Deserialization failure", ex);
}
 
Example 21
@Override
public void onDisconnect(BinaryLogClient client) {
  logger.warn("Disconnect {}@{}", client.getBinlogFilename(), client.getBinlogPosition());
}
 
Example 22
Source Project: datacollector   Source File: GtidSourceOffset.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void positionClient(BinaryLogClient client) {
  client.setGtidSet(gtidSet);
}
 
Example 23
Source Project: datacollector   Source File: BinaryLogConsumer.java    License: Apache License 2.0 4 votes vote down vote up
public BinaryLogConsumer(MysqlSchemaRepository schemaRepository, EventBuffer eventBuffer, BinaryLogClient client) {
  this.schemaRepository = schemaRepository;
  this.eventBuffer = eventBuffer;
  this.client = client;
}
 
Example 24
Source Project: datacollector   Source File: BinLogPositionSourceOffset.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void positionClient(BinaryLogClient client) {
  client.setBinlogFilename(filename);
  client.setBinlogPosition(position);
}
 
Example 25
Source Project: datacollector   Source File: GtidSourceOffset.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void positionClient(BinaryLogClient client) {
  client.setGtidSet(gtidSet);
}
 
Example 26
Source Project: datacollector   Source File: BinLogPositionSourceOffset.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void positionClient(BinaryLogClient client) {
  client.setBinlogFilename(filename);
  client.setBinlogPosition(position);
}
 
Example 27
Source Project: datacollector   Source File: BinaryLogConsumer.java    License: Apache License 2.0 4 votes vote down vote up
public BinaryLogConsumer(MysqlSchemaRepository schemaRepository, EventBuffer eventBuffer, BinaryLogClient client) {
  this.schemaRepository = schemaRepository;
  this.eventBuffer = eventBuffer;
  this.client = client;
}
 
Example 28
Source Project: nifi   Source File: CaptureChangeMySQL.java    License: Apache License 2.0 4 votes vote down vote up
BinaryLogClient createBinlogClient(String hostname, int port, String username, String password) {
    return new BinaryLogClient(hostname, port, username, password);
}
 
Example 29
Source Project: nifi   Source File: BinlogEventListener.java    License: Apache License 2.0 4 votes vote down vote up
public BinlogEventListener(BinaryLogClient client, BlockingQueue<RawBinlogEvent> q) {
    this.client = client;
    this.queue = q;
}
 
Example 30
Source Project: nifi   Source File: BinlogLifecycleListener.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void onConnect(BinaryLogClient binaryLogClient) {
    client.set(binaryLogClient);
    exception.set(null);
}