com.google.api.services.pubsub.PubsubScopes Java Examples

/** Builds a new Pubsub client and returns it. */
public static Pubsub getClient(final HttpTransport httpTransport, final JsonFactory jsonFactory)
    throws IOException {
  checkNotNull(httpTransport);
  checkNotNull(jsonFactory);
  GoogleCredential credential =
      GoogleCredential.getApplicationDefault(httpTransport, jsonFactory);
  if (credential.createScopedRequired()) {
    credential = credential.createScoped(PubsubScopes.all());
  }
  if (credential.getClientAuthentication() != null) {
    System.out.println(
        "\n***Warning! You are not using service account credentials to "
            + "authenticate.\nYou need to use service account credentials for this example,"
            + "\nsince user-level credentials do not have enough pubsub quota,\nand so you will run "
            + "out of PubSub quota very quickly.\nSee "
            + "https://developers.google.com/identity/protocols/application-default-credentials.");
    System.exit(1);
  }
  HttpRequestInitializer initializer = new RetryHttpInitializerWrapper(credential);
  return new Pubsub.Builder(httpTransport, jsonFactory, initializer)
      .setApplicationName(APP_NAME)
      .build();
}
 

/**
     * Setup authorization for local app based on private key.
     * See <a href="https://cloud.google.com/pubsub/configure">cloud.google.com/pubsub/configure</a>
     */
    private void createClient(String private_key_file, String email) throws IOException, GeneralSecurityException {
        HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
        GoogleCredential credential = new GoogleCredential.Builder()
                .setTransport(transport)
                .setJsonFactory(JSON_FACTORY)
                .setServiceAccountScopes(PubsubScopes.all())
                .setServiceAccountId(email)
                .setServiceAccountPrivateKeyFromP12File(new File(private_key_file))
                .build();
        // Please use custom HttpRequestInitializer for automatic retry upon failures.
//        HttpRequestInitializer initializer = new RetryHttpInitializerWrapper(credential);
        pubsub = new Pubsub.Builder(transport, JSON_FACTORY, credential)
                .setApplicationName("eaipubsub")
                .build();
    }
 

/** Builds a new Pubsub client and returns it. */
public static Pubsub getClient(final HttpTransport httpTransport, final JsonFactory jsonFactory)
    throws IOException {
  checkNotNull(httpTransport);
  checkNotNull(jsonFactory);
  GoogleCredential credential =
      GoogleCredential.getApplicationDefault(httpTransport, jsonFactory);
  if (credential.createScopedRequired()) {
    credential = credential.createScoped(PubsubScopes.all());
  }
  if (credential.getClientAuthentication() != null) {
    System.out.println(
        "\n***Warning! You are not using service account credentials to "
            + "authenticate.\nYou need to use service account credentials for this example,"
            + "\nsince user-level credentials do not have enough pubsub quota,\nand so you will run "
            + "out of PubSub quota very quickly.\nSee "
            + "https://developers.google.com/identity/protocols/application-default-credentials.");
    System.exit(1);
  }
  HttpRequestInitializer initializer = new RetryHttpInitializerWrapper(credential);
  return new Pubsub.Builder(httpTransport, jsonFactory, initializer)
      .setApplicationName(APP_NAME)
      .build();
}
 

/**
 * Constructor that will automatically instantiate a Google Cloud Pub/Sub instance.
 *
 * @throws IOException when the initialization of the Google Cloud Pub/Sub client fails.
 */
public GcloudPubsub() throws IOException {
  if (CLOUD_PUBSUB_PROJECT_ID == null) {
    throw new IllegalStateException(GCLOUD_PUBSUB_PROJECT_ID_NOT_SET_ERROR);
  }
  try {
    serverName = InetAddress.getLocalHost().getCanonicalHostName();
  } catch (UnknownHostException e) {
    throw new IllegalStateException("Unable to retrieve the hostname of the system");
  }
  HttpTransport httpTransport = checkNotNull(Utils.getDefaultTransport());
  JsonFactory jsonFactory = checkNotNull(Utils.getDefaultJsonFactory());
  GoogleCredential credential = GoogleCredential.getApplicationDefault(
      httpTransport, jsonFactory);
  if (credential.createScopedRequired()) {
    credential = credential.createScoped(PubsubScopes.all());
  }
  HttpRequestInitializer initializer =
      new RetryHttpInitializerWrapper(credential);
  pubsub = new Pubsub.Builder(httpTransport, jsonFactory, initializer).build();
  logger.info("Google Cloud Pub/Sub Initialization SUCCESS");
}
 

public PubSubClient(PubSubDatastoreProperties datastore, boolean runOnDataflow) {

        Credentials credentials = null;
        if (runOnDataflow || datastore.serviceAccountFile.getValue() == null) {
            try {
                credentials = GoogleCredentials.getApplicationDefault().createScoped(PubsubScopes.all());
            } catch (IOException e) {
                throw TalendRuntimeException.createUnexpectedException(e);
            }
        } else {
            credentials = createCredentials(datastore);
        }
        this.PROJECT_NAME = datastore.projectName.getValue();
        this.client = new Pubsub.Builder(Transport.getTransport(), Transport.getJsonFactory(),
                chainHttpRequestInitializer(credentials,
                        // Do not log 404. It clutters the output and is possibly even required by the caller.
                        new RetryHttpRequestInitializer(ImmutableList.of(404)))).build();
    }
 

/**
 * Builds a new Pubsub client and returns it.
 *
 * @param httpTransport HttpTransport for Pubsub client.
 * @param jsonFactory JsonFactory for Pubsub client.
 * @return Pubsub client.
 * @throws IOException when we can not get the default credentials.
 */
public static Pubsub getClient(final HttpTransport httpTransport,
                               final JsonFactory jsonFactory)
         throws IOException {
    Preconditions.checkNotNull(httpTransport);
    Preconditions.checkNotNull(jsonFactory);
    GoogleCredential credential =
        GoogleCredential.getApplicationDefault(httpTransport, jsonFactory);
    if (credential.createScopedRequired()) {
        credential = credential.createScoped(PubsubScopes.all());
    }
    // Please use custom HttpRequestInitializer for automatic
    // retry upon failures.
    HttpRequestInitializer initializer =
        new RetryHttpInitializerWrapper(credential);
    return new Pubsub.Builder(httpTransport, jsonFactory, initializer)
            .setApplicationName(APP_NAME)
            .build();
}
 

/**
 * Builds a new Pubsub client and returns it.
 *
 * @param httpTransport HttpTransport for Pubsub client.
 * @param jsonFactory JsonFactory for Pubsub client.
 * @return Pubsub client.
 * @throws IOException when we can not get the default credentials.
 */
public static Pubsub getClient(final HttpTransport httpTransport,
                               final JsonFactory jsonFactory)
        throws IOException {
    Preconditions.checkNotNull(httpTransport);
    Preconditions.checkNotNull(jsonFactory);
    GoogleCredential credential = GoogleCredential.getApplicationDefault();
    if (credential.createScopedRequired()) {
        credential = credential.createScoped(PubsubScopes.all());
    }
    // Please use custom HttpRequestInitializer for automatic
    // retry upon failures.
    HttpRequestInitializer initializer =
            new RetryHttpInitializerWrapper(credential);
    return new Pubsub.Builder(httpTransport, jsonFactory, initializer)
            .setApplicationName(APPLICATION_NAME)
            .build();
}
 

PubsubPublisher(Context context, String appname, String project, String topic,
                int credentialResourceId) throws IOException {
    mContext = context;
    mAppname = appname;
    mTopic = "projects/" + project + "/topics/" + topic;

    mHandlerThread = new HandlerThread("pubsubPublisherThread");
    mHandlerThread.start();
    mHandler = new Handler(mHandlerThread.getLooper());

    InputStream jsonCredentials = mContext.getResources().openRawResource(credentialResourceId);
    final GoogleCredential credentials;
    try {
        credentials = GoogleCredential.fromStream(jsonCredentials).createScoped(
                Collections.singleton(PubsubScopes.PUBSUB));
    } finally {
        try {
            jsonCredentials.close();
        } catch (IOException e) {
            Log.e(TAG, "Error closing input stream", e);
        }
    }
    mHandler.post(new Runnable() {
        @Override
        public void run() {
            mHttpTransport = AndroidHttp.newCompatibleTransport();
            JsonFactory jsonFactory = JacksonFactory.getDefaultInstance();
            mPubsub = new Pubsub.Builder(mHttpTransport, jsonFactory, credentials)
                    .setApplicationName(mAppname).build();
        }
    });
}
 

public static Pubsub createPubsubClient(String serviceAccountEmail, String privateKeyFilePath)
    throws IOException, GeneralSecurityException {
  HttpTransport transport = GoogleNetHttpTransport.newTrustedTransport();
  GoogleCredential credential = new GoogleCredential.Builder()
      .setTransport(transport)
      .setJsonFactory(JSON_FACTORY)
      .setServiceAccountScopes(PubsubScopes.all())

      // Obtain this from the "APIs & auth" -> "Credentials"
      // section in the Google Developers Console:
      // https://console.developers.google.com/
      // (and put the e-mail address into your system property obviously)
      .setServiceAccountId(serviceAccountEmail)

      // Download this file from "APIs & auth" -> "Credentials"
      // section in the Google Developers Console:
      // https://console.developers.google.com/
      .setServiceAccountPrivateKeyFromP12File(new File(privateKeyFilePath))
      .build();


  // Please use custom HttpRequestInitializer for automatic
  // retry upon failures.  We provide a simple reference
  // implementation in the "Retry Handling" section.
  HttpRequestInitializer initializer =
      new RetryHttpInitializerWrapper(credential);
  return new Pubsub.Builder(transport, JSON_FACTORY, initializer)
      .setApplicationName("PubSub Example")
      .build();
}
 

public static Credentials createCredentials(PubSubDatastoreProperties datastore) {
    try {
        GoogleCredentials credential = GoogleCredentials
                .fromStream(new FileInputStream(datastore.serviceAccountFile.getValue())).createScoped(PubsubScopes.all());
        return credential;
    } catch (IOException e) {
        throw new RuntimeException("Exception when read service account file: " + datastore.serviceAccountFile.getValue()
                + "\nMessage is:" + e.getMessage());
    }
}
 

public static void main(final String... args) throws IOException, ExecutionException, InterruptedException {

    final String project = Util.defaultProject();

    GoogleCredential credential;

    // Use credentials from file if available
    try {
      credential = GoogleCredential
          .fromStream(new FileInputStream("credentials.json"))
          .createScoped(PubsubScopes.all());
    } catch (IOException e) {
      credential = GoogleCredential.getApplicationDefault()
          .createScoped(PubsubScopes.all());
    }

    LoggingConfigurator.configureDefaults("benchmark", WARN);

    final String subscription = System.getenv("GOOGLE_PUBSUB_SUBSCRIPTION");
    if (subscription == null) {
      System.err.println("Please specify a subscription using the GOOGLE_PUBSUB_SUBSCRIPTION environment variable.");
      System.exit(1);
    }

    System.out.println("Consuming from GOOGLE_PUBSUB_SUBSCRIPTION='" + subscription + "'");

    final Pubsub pubsub = new Pubsub.Builder(Utils.getDefaultTransport(), Utils.getDefaultJsonFactory(), credential)
        .setApplicationName("pull-benchmark")
        .build();

    final Pubsub.Projects.Subscriptions subscriptions = pubsub.projects().subscriptions();

    final String canonicalSubscription = Subscription.canonicalSubscription(project, subscription);

    final ProgressMeter meter = new ProgressMeter();
    final ProgressMeter.Metric receives = meter.group("operations").metric("receives", "messages");

    final ListeningExecutorService executor = MoreExecutors.listeningDecorator(Executors.newCachedThreadPool());

    // Pull concurrently
    for (int i = 0; i < PULLER_CONCURRENCY; i++) {
      pull(subscriptions, canonicalSubscription, receives, executor);
    }
  }
 

public static void main(final String... args) throws IOException {

    final String project = Util.defaultProject();

    GoogleCredential credential;

    // Use credentials from file if available
    try {
      credential = GoogleCredential
          .fromStream(new FileInputStream("credentials.json"))
          .createScoped(PubsubScopes.all());
    } catch (IOException e) {
      credential = GoogleCredential.getApplicationDefault()
          .createScoped(PubsubScopes.all());
    }

    final Pubsub pubsub = Pubsub.builder()
        .credential(credential)
        .compressionLevel(Deflater.BEST_SPEED)
        .enabledCipherSuites(nonGcmCiphers())
        .build();

    final Publisher publisher = Publisher.builder()
        .pubsub(pubsub)
        .concurrency(128)
        .project(project)
        .build();

    LoggingConfigurator.configureDefaults("benchmark", WARN);

    final String topicPrefix = TEST_NAME_PREFIX + ThreadLocalRandom.current().nextInt();

    final List<String> topics = IntStream.range(0, 100)
        .mapToObj(i -> topicPrefix + "-" + i)
        .collect(toList());

    topics.stream()
        .map(topic -> pubsub.createTopic(project, topic))
        .collect(toList())
        .forEach(Futures::getUnchecked);

    final List<Message> messages = IntStream.range(0, 1000)
        .mapToObj(i -> {
          final StringBuilder s = new StringBuilder();
          while (s.length() < MESSAGE_SIZE) {
            s.append(ThreadLocalRandom.current().nextInt());
          }
          return Message.ofEncoded(s.toString());
        })
        .collect(toList());

    final ProgressMeter meter = new ProgressMeter();
    final ProgressMeter.Metric requests = meter.group("operations").metric("publishes", "messages");

    for (int i = 0; i < 100000; i++) {
      benchSend(publisher, messages, topics, requests);
    }
  }
 

public static void main(final String... args) throws IOException, ExecutionException, InterruptedException {

    final String project = Util.defaultProject();

    GoogleCredential credential;

    // Use credentials from file if available
    try {
      credential = GoogleCredential
          .fromStream(new FileInputStream("credentials.json"))
          .createScoped(PubsubScopes.all());
    } catch (IOException e) {
      credential = GoogleCredential.getApplicationDefault()
          .createScoped(PubsubScopes.all());
    }

    final Pubsub pubsub = Pubsub.builder()
        .credential(credential)
        .compressionLevel(Deflater.BEST_SPEED)
        .enabledCipherSuites(nonGcmCiphers())
        .build();

    final Publisher publisher = Publisher.builder()
        .pubsub(pubsub)
        .concurrency(PUBLISHER_CONCURRENCY)
        .project(project)
        .build();

    LoggingConfigurator.configureDefaults("benchmark", WARN);

    final String topic = "test-" + Long.toHexString(ThreadLocalRandom.current().nextLong());
    final String subscription = "test-" + Long.toHexString(ThreadLocalRandom.current().nextLong());

    pubsub.createTopic(project, topic).get();
    pubsub.createSubscription(project, subscription, topic).get();

    final List<String> payloads = IntStream.range(0, 1024)
        .mapToObj(i -> {
          final StringBuilder s = new StringBuilder();
          while (s.length() < MESSAGE_SIZE) {
            s.append(ThreadLocalRandom.current().nextInt());
          }
          return Message.encode(s.toString());
        })
        .collect(toList());
    final int payloadIxMask = 1024 - 1;

    final Supplier<Message> generator = () -> Message.builder()
        .data(payloads.get(ThreadLocalRandom.current().nextInt() & payloadIxMask))
        .putAttribute("ts", Long.toHexString(System.nanoTime()))
        .build();

    final ProgressMeter meter = new ProgressMeter();
    final ProgressMeter.Metric publishes = meter.group("operations").metric("publishes", "messages");
    final ProgressMeter.Metric receives = meter.group("operations").metric("receives", "messages");

    for (int i = 0; i < 100000; i++) {
      publish(publisher, generator, topic, publishes);
    }

    // Pull concurrently and (asynchronously) publish a new message for every message received
    for (int i = 0; i < PULLER_CONCURRENCY; i++) {
      pull(project, pubsub, subscription, receives, () -> publish(publisher, generator, topic, publishes));
    }
  }
 

public static void main(final String... args) throws IOException, ExecutionException, InterruptedException {

    final String project = Util.defaultProject();

    GoogleCredential credential;

    // Use credentials from file if available
    try {
      credential = GoogleCredential
          .fromStream(new FileInputStream("credentials.json"))
          .createScoped(PubsubScopes.all());
    } catch (IOException e) {
      credential = GoogleCredential.getApplicationDefault()
          .createScoped(PubsubScopes.all());
    }

    final Pubsub pubsub = Pubsub.builder()
        .credential(credential)
        .compressionLevel(Deflater.BEST_SPEED)
        .enabledCipherSuites(Util.nonGcmCiphers())
        .build();

    LoggingConfigurator.configureDefaults("benchmark", WARN);

    final String subscription = System.getenv("GOOGLE_PUBSUB_SUBSCRIPTION");
    if (subscription == null) {
      System.err.println("Please specify a subscription using the GOOGLE_PUBSUB_SUBSCRIPTION environment variable.");
      System.exit(1);
    }

    System.out.println("Consuming from GOOGLE_PUBSUB_SUBSCRIPTION='" + subscription + "'");

    final ProgressMeter meter = new ProgressMeter();
    final ProgressMeter.Metric receives = meter.group("operations").metric("receives", "messages");

    final Puller puller = Puller.builder()
        .pubsub(pubsub)
        .batchSize(1000)
        .concurrency(PULLER_CONCURRENCY)
        .project(project)
        .subscription(subscription)
        .messageHandler(new Handler(receives))
        .build();

    while (true) {
      Thread.sleep(1000);
      System.out.println("outstanding messages: " + puller.outstandingMessages());
      System.out.println("outstanding requests: " + puller.outstandingRequests());
    }
  }