com.google.api.gax.batching.FlowControlSettings Java Examples

@Bean
@ConditionalOnMissingBean(name = "publisherBatchSettings")
public BatchingSettings publisherBatchSettings() {
	BatchingSettings.Builder builder = BatchingSettings.newBuilder();

	GcpPubSubProperties.Batching batching = this.gcpPubSubProperties.getPublisher()
			.getBatching();

	FlowControlSettings flowControlSettings = buildFlowControlSettings(batching.getFlowControl());
	if (flowControlSettings != null) {
		builder.setFlowControlSettings(flowControlSettings);
	}

	return ifNotNull(batching.getDelayThresholdSeconds(),
				(x) -> builder.setDelayThreshold(Duration.ofSeconds(x)))
			.apply(ifNotNull(batching.getElementCountThreshold(), builder::setElementCountThreshold)
			.apply(ifNotNull(batching.getEnabled(), builder::setIsEnabled)
			.apply(ifNotNull(batching.getRequestByteThreshold(), builder::setRequestByteThreshold)
			.apply(false)))) ? builder.build() : null;
}
 

private CPSSubscriberTask(StartRequest request, MetricsHandler metricsHandler, int workerCount) {
  this.metricsHandler = metricsHandler;
  ProjectSubscriptionName subscription =
      ProjectSubscriptionName.of(
          request.getProject(), request.getPubsubOptions().getSubscription());
  try {
    this.subscriber =
        Subscriber.newBuilder(subscription, this)
            .setParallelPullCount(workerCount)
            .setFlowControlSettings(
                FlowControlSettings.newBuilder()
                    .setMaxOutstandingElementCount(Long.MAX_VALUE)
                    .setMaxOutstandingRequestBytes(BYTES_PER_WORKER * workerCount)
                    .build())
            .build();
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

/** Return a configured input transform. */
public static Pubsub.Read getInput(Output output) throws IOException {
  // read pubsub messages from INPUT_SUBSCRIPTION
  Pubsub.Read input = new Pubsub.Read(output.env.getString(INPUT_SUBSCRIPTION), output,
      builder -> builder
          .setFlowControlSettings(FlowControlSettings.newBuilder()
              .setMaxOutstandingElementCount(
                  output.type.getMaxOutstandingElementCount(output.env))
              .setMaxOutstandingRequestBytes(
                  output.type.getMaxOutstandingRequestBytes(output.env))
              .build())
          // The number of streaming subscriber connections for reading from Pub/Sub.
          // https://github.com/googleapis/java-pubsub/blob/v1.105.0/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/Subscriber.java#L141
          // https://github.com/googleapis/java-pubsub/blob/v1.105.0/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/Subscriber.java#L318-L320
          // The default number of executor threads is max(6, 2*parallelPullCount).
          // https://github.com/googleapis/java-pubsub/blob/v1.105.0/google-cloud-pubsub/src/main/java/com/google/cloud/pubsub/v1/Subscriber.java#L566-L568
          // Subscriber connections are expected to be CPU bound until flow control thresholds are
          // reached, so parallelism should be no less than the number of available processors.
          .setParallelPullCount(
              output.env.getInt(INPUT_PARALLELISM, Runtime.getRuntime().availableProcessors())),
      getInputCompression(output.env));
  output.env.requireAllVarsUsed();
  // Setup OpenCensus stackdriver exporter after all measurement views have been registered,
  // as seen in https://opencensus.io/exporters/supported-exporters/java/stackdriver-stats/
  StackdriverStatsExporter.createAndRegister();
  return input;
}
 

private FlowControlSettings buildFlowControlSettings(
		GcpPubSubProperties.FlowControl flowControl) {
	FlowControlSettings.Builder builder = FlowControlSettings.newBuilder();

	return ifNotNull(flowControl.getLimitExceededBehavior(), builder::setLimitExceededBehavior)
			.apply(ifNotNull(flowControl.getMaxOutstandingElementCount(),
					builder::setMaxOutstandingElementCount)
			.apply(ifNotNull(flowControl.getMaxOutstandingRequestBytes(),
					builder::setMaxOutstandingRequestBytes)
			.apply(false))) ? builder.build() : null;
}
 

@Bean
@ConditionalOnMissingBean
public SubscriberFactory defaultSubscriberFactory(
		@Qualifier("subscriberExecutorProvider") ExecutorProvider executorProvider,
		@Qualifier("subscriberSystemExecutorProvider")
		ObjectProvider<ExecutorProvider> systemExecutorProvider,
		@Qualifier("subscriberFlowControlSettings")
				ObjectProvider<FlowControlSettings> flowControlSettings,
		@Qualifier("subscriberApiClock") ObjectProvider<ApiClock> apiClock,
		@Qualifier("subscriberRetrySettings") ObjectProvider<RetrySettings> retrySettings,
		TransportChannelProvider transportChannelProvider) {
	DefaultSubscriberFactory factory = new DefaultSubscriberFactory(this.finalProjectIdProvider);
	factory.setExecutorProvider(executorProvider);
	factory.setCredentialsProvider(this.finalCredentialsProvider);
	factory.setHeaderProvider(this.headerProvider);
	factory.setChannelProvider(transportChannelProvider);
	systemExecutorProvider.ifAvailable(factory::setSystemExecutorProvider);
	flowControlSettings.ifAvailable(factory::setFlowControlSettings);
	apiClock.ifAvailable(factory::setApiClock);
	retrySettings.ifAvailable(factory::setSubscriberStubRetrySettings);
	if (this.gcpPubSubProperties.getSubscriber().getMaxAckExtensionPeriod() != null) {
		factory.setMaxAckExtensionPeriod(Duration.ofSeconds(
				this.gcpPubSubProperties.getSubscriber().getMaxAckExtensionPeriod()));
	}
	if (this.gcpPubSubProperties.getSubscriber().getParallelPullCount() != null) {
		factory.setParallelPullCount(
				this.gcpPubSubProperties.getSubscriber().getParallelPullCount());
	}
	if (this.gcpPubSubProperties.getSubscriber()
			.getPullEndpoint() != null) {
		factory.setPullEndpoint(
				this.gcpPubSubProperties.getSubscriber().getPullEndpoint());
	}
	return factory;
}
 

@Test
public void testSubscriberFlowControlSettings() {
	this.contextRunner.run((context) -> {
		FlowControlSettings settings = context
				.getBean("subscriberFlowControlSettings", FlowControlSettings.class);
		assertThat(settings.getMaxOutstandingElementCount()).isEqualTo(17);
		assertThat(settings.getMaxOutstandingRequestBytes()).isEqualTo(18);
		assertThat(settings.getLimitExceededBehavior()).isEqualTo(LimitExceededBehavior.Ignore);
	});
}
 

@Bean
@ConditionalOnMissingBean(name = "subscriberFlowControlSettings")
public FlowControlSettings subscriberFlowControlSettings() {
	return buildFlowControlSettings(
			this.gcpPubSubProperties.getSubscriber().getFlowControl());
}
 

Connection start() {
    log.info("Starting PubSub connection");

    ProjectSubscriptionName subscriptionName = ProjectSubscriptionName
        .of(projectId, subscriptionId);

    FlowControlSettings flowControlSettings =
        FlowControlSettings.newBuilder()
            .setMaxOutstandingElementCount(maxOutstandingElementCount)
            .setMaxOutstandingRequestBytes(maxOutstandingRequestBytes)
            .build();

    ExecutorProvider executorProvider =
        InstantiatingExecutorProvider.newBuilder().setExecutorThreadCount(threads).build();


    log.info("Subscribing to {}", subscriptionName);
    final Receiver receiver = new Receiver(consumer, reporter, errors, consumed);
    subscriber = Subscriber
        .newBuilder(subscriptionName, receiver)
        .setFlowControlSettings(flowControlSettings)
        .setParallelPullCount(threads)
        .setExecutorProvider(executorProvider)
        .setChannelProvider(channelProvider)
        .setCredentialsProvider(credentialsProvider)
        .build();

    subscriber.addListener(
      new Subscriber.Listener() {
          @Override
          public void failed(Subscriber.State from, Throwable failure) {
              // Called when the Subscriber encountered a fatal error and is shutting down
              log.error(
                "An error on subscriber happened (from state: " + from.name() + ")", failure);
              System.exit(1);
          } }, MoreExecutors.directExecutor());


    subscriber.startAsync().awaitRunning();

    log.info("PubSub connection started");
    return this;
}
 

@Override
public void produce(Map<String, String> lastOffsets, int maxBatchSize) throws StageException {
  SynchronousQueue<MessageReplyConsumerBundle> workQueue = new SynchronousQueue<>();

  ProjectSubscriptionName subscriptionName = ProjectSubscriptionName
      .of(
          conf.credentials.projectId,
          conf.subscriptionId
      );

  executor = Executors.newFixedThreadPool(getNumberOfThreads());

  int batchSize = Math.min(maxBatchSize, conf.basic.maxBatchSize);
  if (!getContext().isPreview() && conf.basic.maxBatchSize > maxBatchSize) {
    getContext().reportError(Errors.PUBSUB_10, maxBatchSize);
  }

  for (int i = 0; i < conf.maxThreads; i++) {
    MessageProcessor messageProcessor = new MessageProcessorImpl(
        getContext(),
        batchSize,
        conf.basic.maxWaitTime,
        parserFactory,
        workQueue
    );
    executor.submit(messageProcessor);
    messageProcessors.add(messageProcessor);
  }

  ExecutorProvider executorProvider = InstantiatingExecutorProvider.newBuilder()
      .setExecutorThreadCount(conf.advanced.numThreadsPerSubscriber)
      .build();

  InstantiatingGrpcChannelProvider channelProvider = getChannelProvider();

  FlowControlSettings flowControlSettings = getFlowControlSettings();

  for (int i = 0; i < conf.advanced.numSubscribers; i++) {
    Subscriber s = Subscriber.newBuilder(subscriptionName, new MessageReceiverImpl(workQueue))
        .setCredentialsProvider(credentialsProvider)
        .setExecutorProvider(executorProvider)
        .setChannelProvider(channelProvider)
        .setFlowControlSettings(flowControlSettings)
        .build();
    s.addListener(new Subscriber.Listener() {
      @Override
      public void failed(Subscriber.State from, Throwable failure) {
        LOG.error("Exception thrown in Subscriber: {}", failure.toString(), failure);
        LOG.error("Subscriber state: {}", from.toString());
        Throwables.propagate(failure);
      }
    }, MoreExecutors.directExecutor());
    subscribers.add(s);
  }

  try {
    subscribers.forEach(Subscriber::startAsync);
  } finally {
    LOG.info("Started {} subscribers.", conf.maxThreads);
  }

  while (!getContext().isStopped()) {
    ThreadUtil.sleep(1000);
  }
}
 

/**
 * Returns a flow control setting such that a subscriber will block if it has buffered more messages than can be
 * processed in a single batch times the number of record processors. Since the flow control settings are per
 * subscriber, we should divide by the number of subscribers to avoid buffering too much data in each subscriber.
 *
 * @return settings based on the stage configuration.
 */
private FlowControlSettings getFlowControlSettings() {
  return FlowControlSettings.newBuilder()
      .setLimitExceededBehavior(FlowController.LimitExceededBehavior.Block)
      .setMaxOutstandingElementCount((long) conf.basic.maxBatchSize * conf.maxThreads / conf.advanced.numSubscribers)
      .build();
}
 

/**
 * Set the flow control for the subscribers, including the behaviour for when the flow limits
 * are hit.
 * @param flowControlSettings the flow control settings to set
 */
public void setFlowControlSettings(FlowControlSettings flowControlSettings) {
	this.flowControlSettings = flowControlSettings;
}