com.couchbase.client.dcp.StreamTo Java Examples

The following examples show how to use com.couchbase.client.dcp.StreamTo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CouchbaseStreamingConnectionTest.java    From components with Apache License 2.0 6 votes vote down vote up
@Test
public void testStartStopStreaming() throws InterruptedException {
    Mockito.when(client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW)).thenReturn(Completable.complete());
    Mockito.when(client.startStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    Mockito.when(client.stopStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    SessionState sessionState = Mockito.mock(SessionState.class);
    Mockito.when(sessionState.isAtEnd()).thenReturn(false, true);
    Mockito.when(client.sessionState()).thenReturn(sessionState);

    BlockingQueue<ByteBuf> resultsQueue = new ArrayBlockingQueue<>(4);
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    Mockito.when(client.disconnect()).thenReturn(Completable.complete());


    streamingConnection.startStreaming(resultsQueue);
    streamingConnection.stopStreaming();

    Thread.sleep(1500);
    Mockito.verify(client, Mockito.times(2)).sessionState();
    Mockito.verify(client, Mockito.times(1)).disconnect();
}
 
Example #2
Source File: CouchbaseStreamingConnectionTest.java    From components with Apache License 2.0 6 votes vote down vote up
@Test
public void testStartStreaming() throws InterruptedException {
    Mockito.when(client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW)).thenReturn(Completable.complete());
    Mockito.when(client.startStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    SessionState sessionState = Mockito.mock(SessionState.class);
    Mockito.when(sessionState.isAtEnd()).thenReturn(false, false, true);
    Mockito.when(client.sessionState()).thenReturn(sessionState);

    BlockingQueue<ByteBuf> resultsQueue = new ArrayBlockingQueue<>(3);

    streamingConnection.startStreaming(resultsQueue);

    Assert.assertTrue(streamingConnection.isStreaming());

    Thread.sleep(2000);
    Mockito.verify(client, Mockito.times(3)).sessionState();
}
 
Example #3
Source File: CouchbaseReader.java    From kafka-connect-couchbase with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
  try {
    client.connect().await();

    // Apply the fallback state to all partitions. As of DCP client version 0.12.0,
    // this is the only way to set the sequence number to "now".
    StreamFrom fallbackStreamFrom = streamFrom.withoutSavedOffset();
    client.initializeState(fallbackStreamFrom.asDcpStreamFrom(), StreamTo.INFINITY).await();

    // Overlay any saved offsets (might have saved offsets for only some partitions).
    if (streamFrom.isSavedOffset()) {
      // As of DCP client version 0.12.0, Client.initializeState(BEGINNING, INFINITY)
      // doesn't fetch the failover logs. Do it ourselves to avoid a spurious rollback :-/
      initFailoverLogs();

      // Then restore the partition state and use saved vBucket UUIDs to overlay synthetic failover log entries.
      restoreSavedOffsets();
    }

    client.startStreaming(partitions).await();

  } catch (Throwable t) {
    errorQueue.offer(t);
  }
}
 
Example #4
Source File: PerformanceTestDriver.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
private static void runTest(final Client client, Args args) throws InterruptedException {
  final CountDownLatch latch = new CountDownLatch(args.dcpMessageCount);

  final boolean highLevelApi = Boolean.parseBoolean(args.settings.getProperty("highLevelApi", "false"));
  if (highLevelApi) {
    System.out.println("Using high-level API. Won't be collecting compression metrics.");
    registerHighLevelListeners(latch, client);
  } else {
    System.out.println("Using low-level API.");
    registerLowLevelListeners(latch, client);
  }

  long startNanos = System.nanoTime();
  client.connect().await();
  client.initializeState(StreamFrom.BEGINNING, StreamTo.INFINITY).await();
  client.startStreaming().await();

  latch.await();
  System.out.println("Received at least " + args.dcpMessageCount + " messages. Done!");
  long elapsedNanos = System.nanoTime() - startNanos;

  client.disconnect().await();

  System.out.println("Shutdown complete. Receiving " + args.dcpMessageCount + " DCP events took " +
      TimeUnit.NANOSECONDS.toMillis(elapsedNanos) + " ms");
}
 
Example #5
Source File: CollectionsMutationsIntegrationTest.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Test
public void canStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S").size();
    List<String> collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0");
    Thread.sleep(1000);
    int documentsSize = bucket.upsertOneDocumentToEachCollection(collections, "D", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .collectionAware()
        .start()) {

      //Expect to see all events
      assertStatus(streamer.awaitStreamEnd(), documentsSize, 0, 0);
    }
  }


}
 
Example #6
Source File: CollectionsMutationsIntegrationTest.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Test
public void canStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S");
    List<String> collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0");
    Thread.sleep(1000);
    bucket.upsertOneDocumentToEachCollection(collections, "D-a", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      //Expect to not see any events
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.COLLECTION_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.MUTATIONS);

      int documentsSize = bucket.upsertOneDocumentToEachCollection(collections, "D-b", "S0").size();
      //See only new mutations
      streamer.assertStateCount(documentsSize, DcpStreamer.State.MUTATIONS);
    }
  }
}
 
Example #7
Source File: ScopesIntegrationTest.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Test
public void scopesReconnectsAfterServerRestart() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);

      agent().resetCluster();
      couchbase().restart();

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "B").size();
      //See both A and B scopes
      streamer.assertStateCount((scopesA + scopesB) * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 
Example #8
Source File: CollectionsBasicStreamingIntegrationTest.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Test
public void collectionsStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S-a");
    bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-a", "S-a0");

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.COLLECTION_CREATIONS);

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "S-b").size();
      int collectionsB = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-b", "S-b0").size();

      streamer.assertStateCount(scopesB * VB, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(collectionsB * VB, DcpStreamer.State.COLLECTION_CREATIONS);
    }
  }
}
 
Example #9
Source File: StreamerServiceImpl.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Override
public String start(String bucket, @Default("[]") List<Short> vbuckets, StreamFrom from, StreamTo to, boolean mitigateRollbacks, boolean collectionAware) {
  String streamerId = "dcp-test-streamer-" + nextStreamerId.getAndIncrement();

  Client client = Client.builder()
      .bucket(bucket)
      .credentials(username, password)
      .mitigateRollbacks(mitigateRollbacks ? 100 : 0, TimeUnit.MILLISECONDS)
      .flowControl(1024 * 128)
      .seedNodes(nodes.split(","))
      .userAgent("dcp-integration-test", "0", streamerId)
      .collectionsAware(collectionAware)
      .build();

  idToStreamer.put(streamerId, new DcpStreamer(client, vbuckets, from, to));
  return streamerId;
}
 
Example #10
Source File: ScopesIntegrationTest.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Test
public void scopesStreamFromBeginningToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);

      int scopesB = bucket.createScopes(1, "B").size();
      //See both A and B scopes
      streamer.assertStateCount((scopesA + scopesB) * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 
Example #11
Source File: ScopesIntegrationTest.java    From java-dcp-client with Apache License 2.0 6 votes vote down vote up
@Test
public void scopesStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "B").size();
      //See only B scopes
      streamer.assertStateCount(scopesB * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 
Example #12
Source File: CollectionsBasicStreamingIntegrationTest.java    From java-dcp-client with Apache License 2.0 5 votes vote down vote up
@Test
public void collectionsStreamFromBeginningToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "S-a").size();
    int collectionsA = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-a", "S-a0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .collectionAware()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .start()) {
      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(collectionsA * VB, DcpStreamer.State.COLLECTION_CREATIONS);
    }
  }
}
 
Example #13
Source File: CouchbaseStreamingConnection.java    From components with Apache License 2.0 5 votes vote down vote up
public void startStreaming(final BlockingQueue<ByteBuf> resultsQueue) {
    if (streaming) {
        LOG.warn("This connection already in streaming mode, create another one.");
        return;
    }
    streaming = true;
    this.resultsQueue = resultsQueue;
    client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW).await();
    new Thread(new Runnable() {
        @Override
        public void run() {
            int i = 0;
            try {
                client.startStreaming(partitionsToStream()).await();
                while (true) {
                    if (client.sessionState().isAtEnd()) {
                        break;
                    }
                    try {
                        Thread.sleep(500);
                    } catch (InterruptedException e) {
                        break;
                    }
                }
            } finally {
                streaming = false;
            }
        }
    }, "CouchbaseStreaming-" + threadId.incrementAndGet())
            .start();
}
 
Example #14
Source File: ScopesIntegrationTest.java    From java-dcp-client with Apache License 2.0 5 votes vote down vote up
@Test
public void scopesStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopes = bucket.createScopes(NUMBER_OF_SCOPES, "S").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .collectionAware()
        .start()) {
      assertStatus(streamer.awaitStreamEnd(), 0, 0, 0, scopes * VB, 0, 0, 0, 0);
    }
  }

}
 
Example #15
Source File: CollectionsBasicStreamingIntegrationTest.java    From java-dcp-client with Apache License 2.0 5 votes vote down vote up
@Test
public void collectionsStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopes = bucket.createScopes(NUMBER_OF_SCOPES, "S").size();
    int collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .collectionAware()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .start()) {
      assertStatus(streamer.awaitStreamEnd(), 0, 0, 0, scopes * VB, 0, collections * VB, 0, 0);
    }
  }
}
 
Example #16
Source File: BasicStreamingIntegrationTest.java    From java-dcp-client with Apache License 2.0 5 votes vote down vote up
@Test
public void canStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    final int batchSize = bucket.createOneDocumentInEachVbucket("a").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .start()) {

      streamer.assertStateCount(0, DcpStreamer.State.MUTATIONS);
      bucket.createOneDocumentInEachVbucket("b").size();
      streamer.assertStateCount(batchSize, DcpStreamer.State.MUTATIONS);
    }
  }
}
 
Example #17
Source File: BasicStreamingIntegrationTest.java    From java-dcp-client with Apache License 2.0 5 votes vote down vote up
@Test
public void canStreamFromBeginningToNow() throws Exception {
  couchbase().loadSampleBucket("beer-sample");

  try (RemoteDcpStreamer streamer = newStreamer("beer-sample")
      .range(StreamFrom.BEGINNING, StreamTo.NOW)
      .start()) {

    assertStatus(streamer.awaitStreamEnd(), 7303, 0, 0);
  }
}
 
Example #18
Source File: DcpStreamer.java    From java-dcp-client with Apache License 2.0 5 votes vote down vote up
public Status awaitStreamEnd(long timeout, TimeUnit unit) throws TimeoutException {
  if (this.streamTo == StreamTo.INFINITY) {
    throw new IllegalStateException("Streaming to infinity; can't wait for that!");
  }

  poll().atInterval(100, MILLISECONDS)
      .withTimeout(timeout, unit)
      .until(() -> client.sessionState().isAtEnd());

  return status();
}
 
Example #19
Source File: RemoteAgent.java    From java-dcp-client with Apache License 2.0 4 votes vote down vote up
public StreamBuilder range(StreamFrom from, StreamTo to) {
  this.from = requireNonNull(from);
  this.to = requireNonNull(to);
  return this;
}
 
Example #20
Source File: DcpStreamer.java    From java-dcp-client with Apache License 2.0 4 votes vote down vote up
public DcpStreamer(final Client client, final List<Short> vbuckets,
                   final StreamFrom from, final StreamTo to) {
  this.client = requireNonNull(client);
  this.streamTo = requireNonNull(to);

  client.listener(new DatabaseChangeListener() {
    @Override
    public void onFailure(StreamFailure streamFailure) {
      LOGGER.error("stream failure", streamFailure.getCause());
    }

    @Override
    public void onMutation(Mutation mutation) {
      mutations.getAndIncrement();
      mutation.flowControlAck();
    }

    @Override
    public void onDeletion(Deletion deletion) {
      (deletion.isDueToExpiration() ? expirations : deletions).getAndIncrement();
      deletion.flowControlAck();
    }

    @Override
    public void onScopeCreated(ScopeCreated scopeCreated) {
      scopeCreations.getAndIncrement();
    }

    @Override
    public void onScopeDropped(ScopeDropped scopeDropped) {
      scopeDrops.getAndIncrement();
    }

    @Override
    public void onCollectionCreated(CollectionCreated collectionCreated) {
      collectionCreations.getAndIncrement();
    }

    @Override
    public void onCollectionDropped(CollectionDropped collectionDropped) {
      collectionDrops.getAndIncrement();
    }

    @Override
    public void onCollectionFlushed(CollectionFlushed collectionFlushed) {
      collectionFlushes.getAndIncrement();
    }
  }, FlowControlMode.AUTOMATIC);

  client.connect().await(30, TimeUnit.SECONDS);
  try {
    client.initializeState(from, to).await();
    client.startStreaming(vbuckets.toArray(new Short[0])).await();
  } catch (Throwable t) {
    stop();
    throw t;
  }
}
 
Example #21
Source File: StreamerService.java    From java-dcp-client with Apache License 2.0 2 votes vote down vote up
/**
 * @param bucket Name of the bucket to stream from.
 * @param vbuckets List of vBuckets to stream from, or empty list for all vBuckets.
 * @return ID of the new streamer.
 */
String start(String bucket, @Default("[]") List<Short> vbuckets, StreamFrom from, StreamTo to, boolean mitigateRollbacks, boolean collectionAware);