Java Code Examples for com.couchbase.client.dcp.StreamTo

The following examples show how to use com.couchbase.client.dcp.StreamTo. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: java-dcp-client   Source File: StreamerServiceImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String start(String bucket, @Default("[]") List<Short> vbuckets, StreamFrom from, StreamTo to, boolean mitigateRollbacks, boolean collectionAware) {
  String streamerId = "dcp-test-streamer-" + nextStreamerId.getAndIncrement();

  Client client = Client.builder()
      .bucket(bucket)
      .credentials(username, password)
      .mitigateRollbacks(mitigateRollbacks ? 100 : 0, TimeUnit.MILLISECONDS)
      .flowControl(1024 * 128)
      .seedNodes(nodes.split(","))
      .userAgent("dcp-integration-test", "0", streamerId)
      .collectionsAware(collectionAware)
      .build();

  idToStreamer.put(streamerId, new DcpStreamer(client, vbuckets, from, to));
  return streamerId;
}
 
Example 2
@Test
public void canStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S").size();
    List<String> collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0");
    Thread.sleep(1000);
    int documentsSize = bucket.upsertOneDocumentToEachCollection(collections, "D", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .collectionAware()
        .start()) {

      //Expect to see all events
      assertStatus(streamer.awaitStreamEnd(), documentsSize, 0, 0);
    }
  }


}
 
Example 3
@Test
public void canStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S");
    List<String> collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0");
    Thread.sleep(1000);
    bucket.upsertOneDocumentToEachCollection(collections, "D-a", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      //Expect to not see any events
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.COLLECTION_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.MUTATIONS);

      int documentsSize = bucket.upsertOneDocumentToEachCollection(collections, "D-b", "S0").size();
      //See only new mutations
      streamer.assertStateCount(documentsSize, DcpStreamer.State.MUTATIONS);
    }
  }
}
 
Example 4
@Test
public void collectionsStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    bucket.createScopes(NUMBER_OF_SCOPES, "S-a");
    bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-a", "S-a0");

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(0, DcpStreamer.State.COLLECTION_CREATIONS);

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "S-b").size();
      int collectionsB = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-b", "S-b0").size();

      streamer.assertStateCount(scopesB * VB, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(collectionsB * VB, DcpStreamer.State.COLLECTION_CREATIONS);
    }
  }
}
 
Example 5
Source Project: java-dcp-client   Source File: ScopesIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void scopesStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(0, DcpStreamer.State.SCOPE_CREATIONS);

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "B").size();
      //See only B scopes
      streamer.assertStateCount(scopesB * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 
Example 6
Source Project: java-dcp-client   Source File: ScopesIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void scopesStreamFromBeginningToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);

      int scopesB = bucket.createScopes(1, "B").size();
      //See both A and B scopes
      streamer.assertStateCount((scopesA + scopesB) * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 
Example 7
Source Project: java-dcp-client   Source File: ScopesIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void scopesReconnectsAfterServerRestart() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .collectionAware()
        .start()) {
      int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "A").size();

      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);

      agent().resetCluster();
      couchbase().restart();

      int scopesB = bucket.createScopes(NUMBER_OF_SCOPES, "B").size();
      //See both A and B scopes
      streamer.assertStateCount((scopesA + scopesB) * VB, DcpStreamer.State.SCOPE_CREATIONS);
    }
  }
}
 
Example 8
Source Project: java-dcp-client   Source File: PerformanceTestDriver.java    License: Apache License 2.0 6 votes vote down vote up
private static void runTest(final Client client, Args args) throws InterruptedException {
  final CountDownLatch latch = new CountDownLatch(args.dcpMessageCount);

  final boolean highLevelApi = Boolean.parseBoolean(args.settings.getProperty("highLevelApi", "false"));
  if (highLevelApi) {
    System.out.println("Using high-level API. Won't be collecting compression metrics.");
    registerHighLevelListeners(latch, client);
  } else {
    System.out.println("Using low-level API.");
    registerLowLevelListeners(latch, client);
  }

  long startNanos = System.nanoTime();
  client.connect().await();
  client.initializeState(StreamFrom.BEGINNING, StreamTo.INFINITY).await();
  client.startStreaming().await();

  latch.await();
  System.out.println("Received at least " + args.dcpMessageCount + " messages. Done!");
  long elapsedNanos = System.nanoTime() - startNanos;

  client.disconnect().await();

  System.out.println("Shutdown complete. Receiving " + args.dcpMessageCount + " DCP events took " +
      TimeUnit.NANOSECONDS.toMillis(elapsedNanos) + " ms");
}
 
Example 9
Source Project: kafka-connect-couchbase   Source File: CouchbaseReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
  try {
    client.connect().await();

    // Apply the fallback state to all partitions. As of DCP client version 0.12.0,
    // this is the only way to set the sequence number to "now".
    StreamFrom fallbackStreamFrom = streamFrom.withoutSavedOffset();
    client.initializeState(fallbackStreamFrom.asDcpStreamFrom(), StreamTo.INFINITY).await();

    // Overlay any saved offsets (might have saved offsets for only some partitions).
    if (streamFrom.isSavedOffset()) {
      // As of DCP client version 0.12.0, Client.initializeState(BEGINNING, INFINITY)
      // doesn't fetch the failover logs. Do it ourselves to avoid a spurious rollback :-/
      initFailoverLogs();

      // Then restore the partition state and use saved vBucket UUIDs to overlay synthetic failover log entries.
      restoreSavedOffsets();
    }

    client.startStreaming(partitions).await();

  } catch (Throwable t) {
    errorQueue.offer(t);
  }
}
 
Example 10
@Test
public void testStartStreaming() throws InterruptedException {
    Mockito.when(client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW)).thenReturn(Completable.complete());
    Mockito.when(client.startStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    SessionState sessionState = Mockito.mock(SessionState.class);
    Mockito.when(sessionState.isAtEnd()).thenReturn(false, false, true);
    Mockito.when(client.sessionState()).thenReturn(sessionState);

    BlockingQueue<ByteBuf> resultsQueue = new ArrayBlockingQueue<>(3);

    streamingConnection.startStreaming(resultsQueue);

    Assert.assertTrue(streamingConnection.isStreaming());

    Thread.sleep(2000);
    Mockito.verify(client, Mockito.times(3)).sessionState();
}
 
Example 11
@Test
public void testStartStopStreaming() throws InterruptedException {
    Mockito.when(client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW)).thenReturn(Completable.complete());
    Mockito.when(client.startStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    Mockito.when(client.stopStreaming(Mockito.<Short[]>anyVararg())).thenReturn(Completable.complete());
    SessionState sessionState = Mockito.mock(SessionState.class);
    Mockito.when(sessionState.isAtEnd()).thenReturn(false, true);
    Mockito.when(client.sessionState()).thenReturn(sessionState);

    BlockingQueue<ByteBuf> resultsQueue = new ArrayBlockingQueue<>(4);
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    resultsQueue.put(Mockito.mock(ByteBuf.class));
    Mockito.when(client.disconnect()).thenReturn(Completable.complete());


    streamingConnection.startStreaming(resultsQueue);
    streamingConnection.stopStreaming();

    Thread.sleep(1500);
    Mockito.verify(client, Mockito.times(2)).sessionState();
    Mockito.verify(client, Mockito.times(1)).disconnect();
}
 
Example 12
Source Project: java-dcp-client   Source File: DcpStreamer.java    License: Apache License 2.0 5 votes vote down vote up
public Status awaitStreamEnd(long timeout, TimeUnit unit) throws TimeoutException {
  if (this.streamTo == StreamTo.INFINITY) {
    throw new IllegalStateException("Streaming to infinity; can't wait for that!");
  }

  poll().atInterval(100, MILLISECONDS)
      .withTimeout(timeout, unit)
      .until(() -> client.sessionState().isAtEnd());

  return status();
}
 
Example 13
@Test
public void canStreamFromBeginningToNow() throws Exception {
  couchbase().loadSampleBucket("beer-sample");

  try (RemoteDcpStreamer streamer = newStreamer("beer-sample")
      .range(StreamFrom.BEGINNING, StreamTo.NOW)
      .start()) {

    assertStatus(streamer.awaitStreamEnd(), 7303, 0, 0);
  }
}
 
Example 14
@Test
public void canStreamFromNowToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    final int batchSize = bucket.createOneDocumentInEachVbucket("a").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.NOW, StreamTo.INFINITY)
        .start()) {

      streamer.assertStateCount(0, DcpStreamer.State.MUTATIONS);
      bucket.createOneDocumentInEachVbucket("b").size();
      streamer.assertStateCount(batchSize, DcpStreamer.State.MUTATIONS);
    }
  }
}
 
Example 15
@Test
public void collectionsStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopes = bucket.createScopes(NUMBER_OF_SCOPES, "S").size();
    int collections = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C", "S0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .collectionAware()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .start()) {
      assertStatus(streamer.awaitStreamEnd(), 0, 0, 0, scopes * VB, 0, collections * VB, 0, 0);
    }
  }
}
 
Example 16
@Test
public void collectionsStreamFromBeginningToInfinity() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopesA = bucket.createScopes(NUMBER_OF_SCOPES, "S-a").size();
    int collectionsA = bucket.createCollections(NUMBER_OF_COLLECTIONS, "C-a", "S-a0").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .collectionAware()
        .range(StreamFrom.BEGINNING, StreamTo.INFINITY)
        .start()) {
      streamer.assertStateCount(scopesA * VB, DcpStreamer.State.SCOPE_CREATIONS);
      streamer.assertStateCount(collectionsA * VB, DcpStreamer.State.COLLECTION_CREATIONS);
    }
  }
}
 
Example 17
Source Project: java-dcp-client   Source File: ScopesIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void scopesStreamFromBeginningToNow() throws Exception {
  try (TestBucket bucket = newBucket().create()) {
    int scopes = bucket.createScopes(NUMBER_OF_SCOPES, "S").size();

    try (RemoteDcpStreamer streamer = bucket.newStreamer()
        .range(StreamFrom.BEGINNING, StreamTo.NOW)
        .collectionAware()
        .start()) {
      assertStatus(streamer.awaitStreamEnd(), 0, 0, 0, scopes * VB, 0, 0, 0, 0);
    }
  }

}
 
Example 18
Source Project: components   Source File: CouchbaseStreamingConnection.java    License: Apache License 2.0 5 votes vote down vote up
public void startStreaming(final BlockingQueue<ByteBuf> resultsQueue) {
    if (streaming) {
        LOG.warn("This connection already in streaming mode, create another one.");
        return;
    }
    streaming = true;
    this.resultsQueue = resultsQueue;
    client.initializeState(StreamFrom.BEGINNING, StreamTo.NOW).await();
    new Thread(new Runnable() {
        @Override
        public void run() {
            int i = 0;
            try {
                client.startStreaming(partitionsToStream()).await();
                while (true) {
                    if (client.sessionState().isAtEnd()) {
                        break;
                    }
                    try {
                        Thread.sleep(500);
                    } catch (InterruptedException e) {
                        break;
                    }
                }
            } finally {
                streaming = false;
            }
        }
    }, "CouchbaseStreaming-" + threadId.incrementAndGet())
            .start();
}
 
Example 19
Source Project: java-dcp-client   Source File: DcpStreamer.java    License: Apache License 2.0 4 votes vote down vote up
public DcpStreamer(final Client client, final List<Short> vbuckets,
                   final StreamFrom from, final StreamTo to) {
  this.client = requireNonNull(client);
  this.streamTo = requireNonNull(to);

  client.listener(new DatabaseChangeListener() {
    @Override
    public void onFailure(StreamFailure streamFailure) {
      LOGGER.error("stream failure", streamFailure.getCause());
    }

    @Override
    public void onMutation(Mutation mutation) {
      mutations.getAndIncrement();
      mutation.flowControlAck();
    }

    @Override
    public void onDeletion(Deletion deletion) {
      (deletion.isDueToExpiration() ? expirations : deletions).getAndIncrement();
      deletion.flowControlAck();
    }

    @Override
    public void onScopeCreated(ScopeCreated scopeCreated) {
      scopeCreations.getAndIncrement();
    }

    @Override
    public void onScopeDropped(ScopeDropped scopeDropped) {
      scopeDrops.getAndIncrement();
    }

    @Override
    public void onCollectionCreated(CollectionCreated collectionCreated) {
      collectionCreations.getAndIncrement();
    }

    @Override
    public void onCollectionDropped(CollectionDropped collectionDropped) {
      collectionDrops.getAndIncrement();
    }

    @Override
    public void onCollectionFlushed(CollectionFlushed collectionFlushed) {
      collectionFlushes.getAndIncrement();
    }
  }, FlowControlMode.AUTOMATIC);

  client.connect().await(30, TimeUnit.SECONDS);
  try {
    client.initializeState(from, to).await();
    client.startStreaming(vbuckets.toArray(new Short[0])).await();
  } catch (Throwable t) {
    stop();
    throw t;
  }
}
 
Example 20
Source Project: java-dcp-client   Source File: RemoteAgent.java    License: Apache License 2.0 4 votes vote down vote up
public StreamBuilder range(StreamFrom from, StreamTo to) {
  this.from = requireNonNull(from);
  this.to = requireNonNull(to);
  return this;
}
 
Example 21
Source Project: java-dcp-client   Source File: StreamerService.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * @param bucket Name of the bucket to stream from.
 * @param vbuckets List of vBuckets to stream from, or empty list for all vBuckets.
 * @return ID of the new streamer.
 */
String start(String bucket, @Default("[]") List<Short> vbuckets, StreamFrom from, StreamTo to, boolean mitigateRollbacks, boolean collectionAware);