org.postgresql.replication.LogSequenceNumber Java Examples

The following examples show how to use org.postgresql.replication.LogSequenceNumber. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: App.java    From LogicalDecode with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private LogSequenceNumber getCurrentLSN() throws SQLException
{
    try (Statement st = connection.createStatement())
    {
        try (ResultSet rs = st.executeQuery("select "
                + (((BaseConnection) connection).haveMinimumServerVersion(ServerVersion.v10)
                ? "pg_current_wal_lsn()" : "pg_current_xlog_location()"))) {

            if (rs.next()) {
                String lsn = rs.getString(1);
                return LogSequenceNumber.valueOf(lsn);
            } else {
                return LogSequenceNumber.INVALID_LSN;
            }
        }
    }
}
 
Example #2
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 6 votes vote down vote up
public PostgresWalRecord read() {
  PostgresWalRecord ret = null;
  try {
    ByteBuffer buffer = readNonBlocking();
    if(buffer != null) {
      ret = new PostgresWalRecord(
          buffer,
          getCurrentLSN(),
          configBean.decoderValue
      );
      //sets next LSN
      setNextLSN(LogSequenceNumber.valueOf(ret.getNextLSN()));
    } else {
      LOG.debug("Buffer null");
    }

  } catch (SQLException e) {
    LOG.error("Error reading PostgreSQL replication stream: {}", e.getMessage(), e);
  }
  return ret;
}
 
Example #3
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private LogSequenceNumber getLogSequenceNumber(String startOffset) {
  LogSequenceNumber lsn = null;

  switch(configBean.startValue) {

    case LATEST:
      //startOffset is always NULL when LATEST per PostgresCDCSource.validatePostgresCDCConfigBean()
      if (startOffset == null) {
        startOffset = confirmedFlushLSN;
      }
      LogSequenceNumber lsnStartOffset = LogSequenceNumber.valueOf(startOffset);
      LogSequenceNumber lsnConfirmedFlush = LogSequenceNumber.valueOf(confirmedFlushLSN);
      lsn = lsnStartOffset.asLong() > lsnConfirmedFlush.asLong() ?
          lsnStartOffset : lsnConfirmedFlush;
      break;

    case LSN:
      //startOffset is config.lsn
    case DATE:
      //startOffset is always 1L (which it is earliest avail)

      // is never NULL here
      if (startOffset == null) {
        startOffset = PostgresCDCSource.SEED_LSN;
      }
      lsn = LogSequenceNumber.valueOf(startOffset);
      break;

    default:
      //should throw exception
  }
  return lsn; //never NULL
}
 
Example #4
Source File: TestJdbcPostgresCDCWalRecord.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testWalRecordWal2JsonDecoderBad() {
  String jsonRecord = "I am just a string";

  ByteBuffer bb = ByteBuffer.wrap(jsonRecord.getBytes());

  PostgresWalRecord walRecord = new PostgresWalRecord(
      bb,
      LogSequenceNumber.valueOf("0/0"),
      DecoderValues.WAL2JSON);

  Assert.assertNull(walRecord.getField());
}
 
Example #5
Source File: TestJdbcPostgresCDCWalRecord.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testWalRecordWal2JsonDecoderGood() {
  String jsonRecord = updateTwoTablesManyRows;

  ByteBuffer bb = ByteBuffer.wrap(jsonRecord.getBytes());

  PostgresWalRecord walRecord = new PostgresWalRecord(
      bb,
      LogSequenceNumber.valueOf("0/0"),
      DecoderValues.WAL2JSON);

  Assert.assertNotNull(walRecord.getField());
}
 
Example #6
Source File: TestJdbcPostgresCDCWalRecord.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testWalRecordBadDecoder() {
  String jsonRecord = updateTwoTablesManyRows;

  ByteBuffer bb = ByteBuffer.wrap(jsonRecord.getBytes());

  PostgresWalRecord walRecord = new PostgresWalRecord(
      bb,
      LogSequenceNumber.valueOf("0/0"),
      null);

  Assert.assertNull(walRecord.getField());
}
 
Example #7
Source File: TestJdbcPostgresCDCWalRecord.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testWalRecordFromString() {
  String jsonRecord = updateTwoTablesManyRows;

  ByteBuffer bb = ByteBuffer.wrap(jsonRecord.getBytes());

  PostgresWalRecord walRecord = new PostgresWalRecord(
      bb,
      LogSequenceNumber.valueOf("0/0"),
      DecoderValues.WAL2JSON);

  Assert.assertEquals(jsonRecord + " LSN: 0/0", walRecord.toString());
  Assert.assertEquals(walRecord.getXid(),"598");
  Assert.assertEquals(walRecord.getTimestamp(),"2018-07-13 13:24:44.152109-07");

  List<Field> changes = walRecord.getChanges();
  Assert.assertEquals(changes.size(), 6);

  List<String> validTables = ImmutableList.of("idnames", "hashes");
  changes.forEach(change -> {
    Assert.assertEquals("update",
        PostgresWalRecord.getTypeFromChangeMap(change.getValueAsMap()));
    Assert.assertEquals("public",
        PostgresWalRecord.getSchemaFromChangeMap(change.getValueAsMap()));
    Assert.assertTrue(validTables.contains(
        PostgresWalRecord.getTableFromChangeMap(change.getValueAsMap())));
  });

}
 
Example #8
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 5 votes vote down vote up
public void commitCurrentOffset() throws StageException {
  synchronized (sendUpdatesMutex) {
    LogSequenceNumber lsn = getNextLSN();
    if (lsn != null) {
      LOG.debug("Flushing LSN END: {}", lsn.asString());
      stream.setAppliedLSN(lsn);
      stream.setFlushedLSN(lsn);
      sendUpdates();
    }
  }
}
 
Example #9
Source File: App.java    From LogicalDecode with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public void receiveChangesOccursBeforStartReplication() throws Exception {
    PGConnection pgConnection = (PGConnection) replicationConnection;

    LogSequenceNumber lsn = getCurrentLSN();

    Statement st = connection.createStatement();
    st.execute("insert into test_logical_table(name) values('previous value')");
    st.execute("insert into test_logical_table(name) values('previous value')");
    st.execute("insert into test_logical_table(name) values('previous value')");
    st.close();

    PGReplicationStream stream =
            pgConnection
                    .getReplicationAPI()
                    .replicationStream()
                    .logical()
                    .withSlotName(SLOT_NAME)
                    .withStartPosition(lsn)
                //    .withSlotOption("proto_version",1)
                //    .withSlotOption("publication_names", "pub1")
                   .withSlotOption("include-xids", true)
                //    .withSlotOption("skip-empty-xacts", true)
                    .withStatusInterval(10, TimeUnit.SECONDS)
                    .start();
    ByteBuffer buffer;
    while(true)
    {
        buffer = stream.readPending();
        if (buffer == null) {
            TimeUnit.MILLISECONDS.sleep(10L);
            continue;
        }

        System.out.println( toString(buffer));
        //feedback
        stream.setAppliedLSN(stream.getLastReceiveLSN());
        stream.setFlushedLSN(stream.getLastReceiveLSN());
    }

}
 
Example #10
Source File: PostgreSQLLogPositionManagerTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertUpdateCurrentPosition() {
    PostgreSQLLogPositionManager postgreSQLLogManager = new PostgreSQLLogPositionManager(dataSource);
    WalPosition expected = new WalPosition(LogSequenceNumber.valueOf(POSTGRESQL_96_LSN));
    postgreSQLLogManager.updateCurrentPosition(expected);
    assertThat(postgreSQLLogManager.getCurrentPosition(), is(expected));
}
 
Example #11
Source File: PostgreSQLLogPositionManagerTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertGetCurrentPositionOnPostgreSQL10() throws SQLException {
    PostgreSQLLogPositionManager postgreSQLLogManager = new PostgreSQLLogPositionManager(dataSource);
    when(databaseMetaData.getDatabaseMajorVersion()).thenReturn(10);
    WalPosition actual = postgreSQLLogManager.getCurrentPosition();
    assertThat(actual.getLogSequenceNumber(), is(LogSequenceNumber.valueOf(POSTGRESQL_10_LSN)));
}
 
Example #12
Source File: TestDecodingPluginTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertDecodeWriteRowEventWithByteA() {
    LogSequenceNumber lsn = LogSequenceNumber.valueOf("0/14EFDB8");
    ByteBuffer data = ByteBuffer.wrap("table public.test: INSERT: data[bytea]:'\\xff00ab'".getBytes());
    WriteRowEvent actual = (WriteRowEvent) new TestDecodingPlugin(null).decode(data, lsn);
    assertThat(actual.getLogSequenceNumber(), is(lsn));
    assertThat(actual.getTableName(), is("test"));
    assertThat(actual.getAfterRow().get(0), is(new byte[] {(byte) 0xff, (byte) 0, (byte) 0xab}));
}
 
Example #13
Source File: TestDecodingPlugin.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Override
public AbstractWalEvent decode(final ByteBuffer data, final LogSequenceNumber logSequenceNumber) {
    AbstractWalEvent result;
    String eventType = readEventType(data);
    if ("table".equals(eventType)) {
        result = readTableEvent(data);
    } else {
        result = new PlaceholderEvent();
    }
    result.setLogSequenceNumber(logSequenceNumber);
    return result;
}
 
Example #14
Source File: LogicalReplication.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
/**
 * Create PostgreSQL replication stream.
 *
 * @param pgConnection PostgreSQL connection
 * @param slotName slot name
 * @param startPosition start position
 * @return replication stream
 * @throws SQLException sql exception
 */
public PGReplicationStream createReplicationStream(final PGConnection pgConnection, final String slotName, final LogSequenceNumber startPosition) throws SQLException {
    return pgConnection.getReplicationAPI()
            .replicationStream()
            .logical()
            .withStartPosition(startPosition)
            .withSlotName(slotName)
            .withSlotOption("include-xids", true)
            .withSlotOption("skip-empty-xacts", true)
            .start();
}
 
Example #15
Source File: PostgreSQLLogPositionManager.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
private WalPosition getCurrentLsn(final Connection connection) throws SQLException {
    String sql;
    if (9 == connection.getMetaData().getDatabaseMajorVersion() && 6 <= connection.getMetaData().getDatabaseMinorVersion()) {
        sql = "SELECT PG_CURRENT_XLOG_LOCATION()";
    } else if (10 <= connection.getMetaData().getDatabaseMajorVersion()) {
        sql = "SELECT PG_CURRENT_WAL_LSN()";
    } else {
        throw new RuntimeException("Not support PostgreSQL version:" + connection.getMetaData().getDatabaseProductVersion());
    }
    PreparedStatement ps = connection.prepareStatement(sql);
    ResultSet rs = ps.executeQuery();
    rs.next();
    return new WalPosition(LogSequenceNumber.valueOf(rs.getString(1)));
}
 
Example #16
Source File: TestDecodingPluginTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertDecodeWriteRowEvent() {
    LogSequenceNumber lsn = LogSequenceNumber.valueOf("0/14EFDB8");
    ByteBuffer data = ByteBuffer.wrap("table public.test: INSERT: data[character varying]:'1 2 3'''".getBytes());
    WriteRowEvent actual = (WriteRowEvent) new TestDecodingPlugin(null).decode(data, lsn);
    assertThat(actual.getLogSequenceNumber(), is(lsn));
    assertThat(actual.getTableName(), is("test"));
    assertThat(actual.getAfterRow().get(0), is("1 2 3'"));
}
 
Example #17
Source File: TestDecodingPluginTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertDecodeUpdateRowEvent() {
    LogSequenceNumber lsn = LogSequenceNumber.valueOf("0/14EFDB8");
    ByteBuffer data = ByteBuffer.wrap("table public.test: UPDATE: data[character varying]:'1 2 3'''".getBytes());
    UpdateRowEvent actual = (UpdateRowEvent) new TestDecodingPlugin(null).decode(data, lsn);
    assertThat(actual.getLogSequenceNumber(), is(lsn));
    assertThat(actual.getTableName(), is("test"));
    assertThat(actual.getAfterRow().get(0), is("1 2 3'"));
}
 
Example #18
Source File: TestDecodingPluginTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertDecodeDeleteRowEvent() {
    LogSequenceNumber lsn = LogSequenceNumber.valueOf("0/14EFDB8");
    ByteBuffer data = ByteBuffer.wrap("table public.test: DELETE: data[integer]:1".getBytes());
    DeleteRowEvent actual = (DeleteRowEvent) new TestDecodingPlugin(null).decode(data, lsn);
    assertThat(actual.getLogSequenceNumber(), is(lsn));
    assertThat(actual.getTableName(), is("test"));
    assertThat(actual.getPrimaryKeys().get(0), is(1));
}
 
Example #19
Source File: PostgreSQLLogPositionManagerTest.java    From shardingsphere with Apache License 2.0 5 votes vote down vote up
@Test
public void assertGetCurrentPositionOnPostgreSQL96() throws SQLException {
    PostgreSQLLogPositionManager postgreSQLLogManager = new PostgreSQLLogPositionManager(dataSource);
    when(databaseMetaData.getDatabaseMajorVersion()).thenReturn(9);
    when(databaseMetaData.getDatabaseMinorVersion()).thenReturn(6);
    WalPosition actual = postgreSQLLogManager.getCurrentPosition();
    assertThat(actual.getLogSequenceNumber(), is(LogSequenceNumber.valueOf(POSTGRESQL_96_LSN)));
}
 
Example #20
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public LogSequenceNumber getCurrentLSN() {
  return stream.getLastReceiveLSN();
}
 
Example #21
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public LogSequenceNumber getNextLSN() {
  return nextLSN;
}
 
Example #22
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public void setNextLSN(LogSequenceNumber nextLSN) {
  this.nextLSN = nextLSN;
}
 
Example #23
Source File: PostgresWalRecord.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public LogSequenceNumber getLsn() {
  return lsn;
}
 
Example #24
Source File: PostgresWalRecord.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public void setLsn(LogSequenceNumber lsn) {
  this.lsn = lsn;
}
 
Example #25
Source File: PostgresWalRecord.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public PostgresWalRecord(ByteBuffer buffer, LogSequenceNumber lsn, DecoderValues decoder) {
  this.buffer = buffer;
  this.lsn = lsn;
  this.field = null; //converter throws exception best handled in getter
  this.decoder = decoder;
}
 
Example #26
Source File: PostgresCDCWalReceiver.java    From datacollector with Apache License 2.0 4 votes vote down vote up
public LogSequenceNumber createReplicationStream(String startOffset)
    throws StageException, InterruptedException, TimeoutException, SQLException {

  boolean newSlot = false;
  if (!doesReplicationSlotExists(slotName)) {
    createReplicationSlot(slotName);
    newSlot = true;
  }
  obtainReplicationSlotInfo(slotName);

  connection = getConnection(this.uri, this.properties);
  PGConnection pgConnection = connection.unwrap(PGConnection.class);

  ChainedLogicalStreamBuilder streamBuilder = pgConnection
      .getReplicationAPI()
      .replicationStream()
      .logical()
      .withSlotName(slotName)
      .withSlotOption("include-xids", true)
      .withSlotOption("include-timestamp", true)
      .withSlotOption("include-lsn", true);

  LogSequenceNumber streamLsn;
  LogSequenceNumber serverFlushedLsn = LogSequenceNumber.valueOf(confirmedFlushLSN);
  if (newSlot) {
    //if the replication slot was just created setting the start offset to an older LSN is a NO OP
    //setting it to a future LSN is risky as the LSN could be invalid (we have to consider the LSN an opaque value).
    //we set the offset then to the 'confirmed_flush_lsn' of the replication slot, that happens to be the
    //the starting point of the newly created replication slot.
    //
    //NOTE that the DATE filter, if a date in the future, it will work as expected because we filter  by DATE.
    streamLsn = serverFlushedLsn;
  } else {

    switch (configBean.startValue) {
      case LATEST:
        // we pick up what is in the replication slot
        streamLsn = serverFlushedLsn;
        break;
      case LSN:
      case DATE:
        LogSequenceNumber configStartLsn = LogSequenceNumber.valueOf(startOffset);
        if (configStartLsn.asLong() > serverFlushedLsn.asLong()) {
          // the given LSN is newer than the last flush, we can safely forward the stream to it,
          // referenced data (by the given configStartLsn should be there)
          streamLsn = configStartLsn;
        } else {
          // we ignore the config start LSN as it is before the last flushed, not in the server anymore
          // this is the normal scenario on later pipeline restarts
          streamLsn = serverFlushedLsn;
          LOG.debug(
              "Configuration Start LSN '{}' is older than server Flushed LSN '{}', this is expected after the first pipeline run",
              configStartLsn,
              serverFlushedLsn
          );
        }
        break;
      default:
        throw new IllegalStateException("Should not happen startValue enum not handled" + configBean.startValue);
    }
  }
  streamBuilder.withStartPosition(streamLsn);

  stream = streamBuilder.start();

  LOG.debug("Starting the Stream with LSN : {}", streamLsn);

  heartBeatSender.scheduleAtFixedRate(this::sendUpdates, 1, 900, TimeUnit.MILLISECONDS);
  return streamLsn;
}
 
Example #27
Source File: PostgresCDCSource.java    From datacollector with Apache License 2.0 4 votes vote down vote up
private Optional<List<ConfigIssue>> validatePostgresCDCConfigBean(PostgresCDCConfigBean configBean) {
  List<ConfigIssue> issues = new ArrayList<>();

  if (configBean.minVersion == null) {
    this.getConfigBean().minVersion = PgVersionValues.NINEFOUR;
  }

  if (configBean.decoderValue == null) {
    this.getConfigBean().decoderValue = DecoderValues.WAL2JSON;
  }

  if (configBean.replicationType == null ) {
    this.getConfigBean().replicationType = "database";
  }

  if (TimeUnit.SECONDS.toMillis(configBean.pollInterval) > configBean.maxBatchWaitTime) {
    issues.add(
        getContext().createConfigIssue(
            Groups.CDC.name(),
            "postgresCDCConfigBean.pollInterval",
            JdbcErrors.JDBC_412, configBean.pollInterval, configBean.maxBatchWaitTime)
    );
  }

  switch(configBean.startValue) {

    case LSN:
      //Validate startLSN
      if (configBean.startLSN == null ||
          configBean.startLSN.isEmpty() ||
          (LogSequenceNumber.valueOf(configBean.startLSN).equals(LogSequenceNumber.INVALID_LSN))
      ) {
        issues.add(
            getContext().createConfigIssue(Groups.CDC.name(),
                "postgresCDCConfigBean.startLSN",
                JdbcErrors.JDBC_408)
        );
      } else {
        this.setConfigInitialOffset(configBean.startLSN);
      }
      break;

    case DATE:
      //Validate startDate
      zoneId = ZoneId.of(configBean.dbTimeZone);
      dateTimeColumnHandler = new DateTimeColumnHandler(zoneId, configBean.convertTimestampToString);
      try {
        startDate = LocalDateTime.parse(
            configBean.startDate, DateTimeFormatter.ofPattern("MM-dd-yyyy HH:mm:ss")
        );
        /* Valid offset that should be as early as possible to get the most number of WAL
        records available for the date filter to process. */
        this.setConfigInitialOffset(LogSequenceNumber.valueOf(1L).asString());
      } catch (DateTimeParseException e) {
        issues.add(
            getContext().createConfigIssue(
                Groups.CDC.name(),
                "postgresCDCConfigBean.startDate",
                JdbcErrors.JDBC_408
            )
        );
      }
      break;

    case LATEST:
      this.setConfigInitialOffset(null); //Null picks up the latestLSN.
      break;

    default:
      //Should never happen
      issues.add(
          getContext().createConfigIssue(
              Groups.CDC.name(),
              "postgresCDCConfigBean.startValue",
              JdbcErrors.JDBC_408
          )
      );
  }

  return Optional.ofNullable(issues);
}
 
Example #28
Source File: TestWalRecordFilteringUtils.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Test
public void testPassesTableFilter() {
  /* Set up valid schemasAndTables that have already been pre-filtered
      in walReceiver. This List will be returned from mock class
   */
  schemasAndTables = new ArrayList<SchemaAndTable>() {{
    add(new SchemaAndTable("public", "table1"));
    add(new SchemaAndTable("public", "table2"));
    add(new SchemaAndTable("public", "table3"));
  }};

  /* Setup change in WAL record that will be tested against table filter */
  final Map<String, Field> change1 = new HashMap<String, Field> () {{
    put("kind", Field.create("update"));
    put("schema", Field.create("public"));
    put("table", Field.create("table1"));
  }};

  final Map<String, Field> change2 = new HashMap<String, Field> () {{
    put("kind", Field.create("update"));
    put("schema", Field.create("public"));
    put("table", Field.create("not_a_valid_table"));
  }};

  final Map<String, Field> change3 = new HashMap<String, Field> () {{
    put("kind", Field.create("update"));
    put("schema", Field.create("public"));
    put("table", Field.create("table3"));
  }};

  List<Field> changes = new ArrayList<Field>() {{
    add(Field.create(change1));
    add(Field.create(change2));
    add(Field.create(change3));
  }};

  TestPgMockCDCRecord testPgMockCDCRecord = new TestPgMockCDCRecord("511", "0/0",
      "2018-07-09 10:16:23.815-07", changes);

  //Setting startValue to latest means filter only checks tables, not dates
  configBean.startValue = StartValues.LATEST;
  Mockito.when(pgSourceMock.getWalReceiver()).thenReturn(walReceiverMock);
  Mockito.when(pgSourceMock.getConfigBean()).thenReturn(configBean);

  Mockito.when(walReceiverMock.getSchemasAndTables()).thenReturn(schemasAndTables);
  Mockito.when(walRecordMock.getChanges()).thenReturn(testPgMockCDCRecord.getCDCRecordChanges());
  Mockito.when(walRecordMock.getBuffer()).thenReturn(ByteBuffer.wrap(new String("test").getBytes()));
  Mockito.when(walRecordMock.getLsn()).thenReturn(LogSequenceNumber.valueOf("0/0"));
  Mockito.when(walRecordMock.getDecoder()).thenReturn(DecoderValues.WAL2JSON);
  Mockito.when(walRecordMock.getField()).thenReturn(Field.create(testPgMockCDCRecord.getCDCRecord()));

  PostgresWalRecord filteredRecord = WalRecordFilteringUtils.passesTableFilter(walRecordMock, pgSourceMock);
  Assert.assertEquals(walRecordMock.getChanges().size(), 3);
  Assert.assertEquals(filteredRecord.getChanges().size(), 2);
}
 
Example #29
Source File: TestWalRecordFilteringUtils.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Test
public void testPassesDateFilter() {

  ZoneId zoneId =  ZoneId.of("America/Los_Angeles");

  /* TEST - not testing records based on date, getting StartValues.LATEST, should pass */

  // Setting table/schema filter to null to passesTableFilter() not tested here
  schemasAndTables = null;
  /* Setup change in WAL record that will be tested against table filter */
  final Map<String, Field> change1 = new HashMap<String, Field> () {{
    put("kind", Field.create("update"));
    put("schema", Field.create("public"));
    put("table", Field.create("table1"));
  }};

  List<Field> changes = new ArrayList<Field>() {{
    add(Field.create(change1));
  }};

  TestPgMockCDCRecord testPgMockCDCRecord = new TestPgMockCDCRecord("511", "0/0",
      "2018-07-09 10:16:23.815-07", changes);

  //Setting startValue to latest means not checking dates - automatic pass
  configBean.startValue = StartValues.LATEST;
  Mockito.when(pgSourceMock.getWalReceiver()).thenReturn(walReceiverMock);
  Mockito.when(pgSourceMock.getConfigBean()).thenReturn(configBean);
  Mockito.when(walReceiverMock.getSchemasAndTables()).thenReturn(schemasAndTables);
  Mockito.when(walRecordMock.getChanges()).thenReturn(testPgMockCDCRecord.getCDCRecordChanges());
  Mockito.when(walRecordMock.getBuffer()).thenReturn(ByteBuffer.wrap("test".getBytes()));
  Mockito.when(walRecordMock.getLsn()).thenReturn(LogSequenceNumber.valueOf("0/0"));
  Mockito.when(walRecordMock.getXid()).thenReturn("511");
  Mockito.when(walRecordMock.getDecoder()).thenReturn(DecoderValues.WAL2JSON);
  Mockito.when(walRecordMock.getField()).thenReturn(Field.create(testPgMockCDCRecord.getCDCRecord()));

  PostgresWalRecord filteredRecord = WalRecordFilteringUtils.passesStartValueFilter(walRecordMock, pgSourceMock);
  Assert.assertEquals(walRecordMock.getChanges().size(), 1);
  Assert.assertEquals(filteredRecord.getChanges().size(), 1);

  /* TEST - testing recordDate > filterDate so should pass */

  Mockito.when(pgSourceMock.getWalReceiver()).thenReturn(walReceiverMock);
  Mockito.when(walReceiverMock.getSchemasAndTables()).thenReturn(schemasAndTables);
  Mockito.when(walRecordMock.getChanges()).thenReturn(testPgMockCDCRecord.getCDCRecordChanges());
  Mockito.when(walRecordMock.getTimestamp()).thenReturn(testPgMockCDCRecord.getTimeStamp());
  Mockito.when(walRecordMock.getLsn()).thenReturn(LogSequenceNumber.valueOf("0/0"));
  Mockito.when(walRecordMock.getXid()).thenReturn("511");
  Mockito.when(walRecordMock.getDecoder()).thenReturn(DecoderValues.WAL2JSON);
  Mockito.when(walRecordMock.getField()).thenReturn(Field.create(testPgMockCDCRecord.getCDCRecord()));

  configBean.startValue = StartValues.DATE;
  Mockito.when(pgSourceMock.getConfigBean()).thenReturn(configBean);
  Mockito.when(pgSourceMock.getStartDate()).thenReturn(LocalDateTime.parse("2000-01-01 01:00:00",
      DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));
  Mockito.when(pgSourceMock.getZoneId()).thenReturn(zoneId);

  filteredRecord = WalRecordFilteringUtils.passesStartValueFilter(walRecordMock, pgSourceMock);
  Assert.assertEquals(walRecordMock.getChanges().size(), 1);
  Assert.assertEquals(filteredRecord.getChanges().size(), 1);

  /* TEST - testing recordDate < filterDate so should fail */

  Mockito.when(pgSourceMock.getWalReceiver()).thenReturn(walReceiverMock);
  Mockito.when(walReceiverMock.getSchemasAndTables()).thenReturn(schemasAndTables);
  Mockito.when(walRecordMock.getChanges()).thenReturn(testPgMockCDCRecord.getCDCRecordChanges());
  Mockito.when(walRecordMock.getLsn()).thenReturn(LogSequenceNumber.valueOf("0/0"));
  Mockito.when(walRecordMock.getXid()).thenReturn("511");
  Mockito.when(walRecordMock.getDecoder()).thenReturn(DecoderValues.WAL2JSON);
  Mockito.when(walRecordMock.getField()).thenReturn(Field.create(testPgMockCDCRecord.getCDCRecord()));

  configBean.startValue = StartValues.DATE;
  Mockito.when(pgSourceMock.getConfigBean()).thenReturn(configBean);
  Mockito.when(pgSourceMock.getStartDate()).thenReturn(LocalDateTime.parse("2020-01-01 01:00:00",
      DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")));

  filteredRecord = WalRecordFilteringUtils.passesStartValueFilter(walRecordMock, pgSourceMock);
  Assert.assertNull(filteredRecord);
}
 
Example #30
Source File: DecodingPlugin.java    From shardingsphere with Apache License 2.0 2 votes vote down vote up
/**
 * Decode wal event from logical replication data.
 *
 * @param data              of logical replication
 * @param logSequenceNumber wal lsn
 * @return wal event
 */
AbstractWalEvent decode(ByteBuffer data, LogSequenceNumber logSequenceNumber);