com.google.cloud.storage.Storage Java Examples

The following examples show how to use com.google.cloud.storage.Storage. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: QuickStartIT.java    From java-docs-samples with Apache License 2.0 6 votes vote down vote up
private static final void deleteObjects() {
  Storage storage = StorageOptions.getDefaultInstance().getService();
  for (BlobInfo info :
      storage
          .list(
              bucketName,
              BlobListOption.versions(true),
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(path + "/"))
          .getValues()) {
    storage.delete(info.getBlobId());
  }
}
 
Example #2
Source File: EnableLifecycleManagement.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
public static void enableLifecycleManagement(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);

  // See the LifecycleRule documentation for additional info on what you can do with lifecycle
  // management rules. This one deletes objects that are over 100 days old.
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/BucketInfo.LifecycleRule.html
  bucket
      .toBuilder()
      .setLifecycleRules(
          ImmutableList.of(
              new LifecycleRule(
                  LifecycleRule.LifecycleAction.newDeleteAction(),
                  LifecycleRule.LifecycleCondition.newBuilder().setAge(100).build())))
      .build()
      .update();

  System.out.println("Lifecycle management was enabled and configured for bucket " + bucketName);
}
 
Example #3
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of setting a default KMS key on a bucket. */
public Bucket setDefaultKmsKey(String bucketName, String kmsKeyName) throws StorageException {
  // [START storage_set_bucket_default_kms_key]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of the existing bucket to set a default KMS key for, e.g. "my-bucket"
  // String bucketName = "my-bucket"

  // The name of the KMS-key to use as a default
  // Key names are provided in the following format:
  // 'projects/<PROJECT>/locations/<LOCATION>/keyRings/<RING_NAME>/cryptoKeys/<KEY_NAME>'
  // String kmsKeyName = ""

  BucketInfo bucketInfo =
      BucketInfo.newBuilder(bucketName).setDefaultKmsKeyName(kmsKeyName).build();

  Bucket bucket = storage.update(bucketInfo);

  System.out.println("Default KMS Key Name: " + bucket.getDefaultKmsKeyName());
  // [END storage_set_bucket_default_kms_key]
  return bucket;
}
 
Example #4
Source File: PutGCSObjectTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAclAttributeProject() throws Exception {
    reset(storage, blob);
    final PutGCSObject processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.create(any(BlobInfo.class), any(InputStream.class), any(Storage.BlobWriteOption.class)))
            .thenReturn(blob);

    final Acl.Project mockProject = mock(Acl.Project.class);
    when(mockProject.getProjectId()).thenReturn(OWNER_PROJECT_ID);
    when(blob.getOwner()).thenReturn(mockProject);

    runner.enqueue("test");
    runner.run();

    runner.assertAllFlowFilesTransferred(PutGCSObject.REL_SUCCESS);
    runner.assertTransferCount(PutGCSObject.REL_SUCCESS, 1);

    final MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(PutGCSObject.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(OWNER_ATTR, OWNER_PROJECT_ID);
    mockFlowFile.assertAttributeEquals(OWNER_TYPE_ATTR, "project");
}
 
Example #5
Source File: StorageExample.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
private void run(Storage storage, Path uploadFrom, BlobInfo blobInfo) throws IOException {
  if (Files.size(uploadFrom) > 1_000_000) {
    // When content is not available or large (1MB or more) it is recommended
    // to write it in chunks via the blob's channel writer.
    try (WriteChannel writer = storage.writer(blobInfo)) {
      byte[] buffer = new byte[1024];
      try (InputStream input = Files.newInputStream(uploadFrom)) {
        int limit;
        while ((limit = input.read(buffer)) >= 0) {
          try {
            writer.write(ByteBuffer.wrap(buffer, 0, limit));
          } catch (Exception ex) {
            ex.printStackTrace();
          }
        }
      }
    }
  } else {
    byte[] bytes = Files.readAllBytes(uploadFrom);
    // create the blob in one request.
    storage.create(blobInfo, bytes);
  }
  System.out.println("Blob was created");
}
 
Example #6
Source File: GcsStreamingMessageSourceTests.java    From spring-cloud-gcp with Apache License 2.0 6 votes vote down vote up
@Bean
public Storage gcsClient() {
	Storage gcs = mock(Storage.class);

	willAnswer((invocationOnMock) ->
		new PageImpl<>(null, null,
				Stream.of(
						createBlob("gcsbucket", "gamma"),
						createBlob("gcsbucket", "beta"),
						createBlob("gcsbucket", "alpha/alpha"))
						.collect(Collectors.toList())))
			.given(gcs).list(eq("gcsbucket"));

	willAnswer((invocationOnMock) -> mock(ReadChannel.class))
			.given(gcs).reader(eq("gcsbucket"), eq("alpha/alpha"));
	willAnswer((invocationOnMock) -> mock(ReadChannel.class))
			.given(gcs).reader(eq("gcsbucket"), eq("beta"));
	willAnswer((invocationOnMock) -> mock(ReadChannel.class))
			.given(gcs).reader(eq("gcsbucket"), eq("gamma"));

	return gcs;
}
 
Example #7
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of how to set event-based hold for a blob */
public Blob setEventBasedHold(String bucketName, String blobName) throws StorageException {
  // [START storage_set_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  // The name of a blob, e.g. "my-blob"
  // String blobName = "my-blob";

  BlobId blobId = BlobId.of(bucketName, blobName);
  Blob blob = storage.update(BlobInfo.newBuilder(blobId).setEventBasedHold(true).build());

  System.out.println("Event-based hold was set for " + blobName);
  // [END storage_set_event_based_hold]
  return blob;
}
 
Example #8
Source File: ListGCSBucketTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testYieldOnBadStateRestore() throws Exception {
    reset(storage, mockBlobPage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Iterable<Blob> mockList = ImmutableList.of();

    when(mockBlobPage.getValues()).thenReturn(mockList);
    when(mockBlobPage.getNextPage()).thenReturn(null);
    when(storage.list(anyString(), any(Storage.BlobListOption[].class))).thenReturn(mockBlobPage);

    runner.getStateManager().setFailOnStateGet(Scope.CLUSTER, true);
    runner.enqueue("test");
    runner.run();

    runner.assertTransferCount(ListGCSBucket.REL_SUCCESS, 0);
    assertEquals(1, runner.getLogger().getErrorMessages().size());
}
 
Example #9
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of how to disable default event-based hold for a bucket */
public Bucket disableDefaultEventBasedHold(String bucketName) throws StorageException {
  // [START storage_disable_default_event_based_hold]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  Bucket bucket =
      storage.update(BucketInfo.newBuilder(bucketName).setDefaultEventBasedHold(false).build());

  System.out.println("Default event-based hold was disabled for " + bucketName);
  // [END storage_disable_default_event_based_hold]
  return bucket;
}
 
Example #10
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of how to enable uniform bucket-level access for a bucket */
public Bucket enableUniformBucketLevelAccess(String bucketName) throws StorageException {
  // [START storage_enable_uniform_bucket_level_access]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The name of a bucket, e.g. "my-bucket"
  // String bucketName = "my-bucket";

  BucketInfo.IamConfiguration iamConfiguration =
      BucketInfo.IamConfiguration.newBuilder().setIsUniformBucketLevelAccessEnabled(true).build();
  Bucket bucket =
      storage.update(
          BucketInfo.newBuilder(bucketName).setIamConfiguration(iamConfiguration).build());

  System.out.println("Uniform bucket-level access was enabled for " + bucketName);
  // [END storage_enable_uniform_bucket_level_access]
  return bucket;
}
 
Example #11
Source File: StorageSnippets.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
/** Example of deleting an existing inactive HMAC key. * */
public void deleteHmacKey(String accessId, String projectId) throws StorageException {
  // [START storage_delete_hmac_key]
  // Instantiate a Google Cloud Storage client
  Storage storage = StorageOptions.getDefaultInstance().getService();

  // The access ID of the HMAC key, e.g. "GOOG0234230X00"
  // String accessId = "GOOG0234230X00";
  //
  // The ID of the project to which the service account belongs.
  // String projectId = "project-id";
  HmacKeyMetadata metadata =
      storage.getHmacKey(accessId, Storage.GetHmacKeyOption.projectId(projectId));
  storage.deleteHmacKey(metadata);

  System.out.println(
      "The key is deleted, though it will still appear in getHmacKeys() results given showDeletedKey is true.");
  // [END storage_delete_hmac_key]
}
 
Example #12
Source File: GcsObjectPostProcessingHandler.java    From datacollector with Apache License 2.0 6 votes vote down vote up
/**
 * Copy a blob for gcs to a destination bucket and a path (and delete the source blob if needed)
 * @param sourceBlobId sourceBlobId
 * @param destinationBucket destination bucket
 * @param destinationPath destination path
 * @param deleteSource delete source blob
 */
private void copy(BlobId sourceBlobId, String destinationBucket, String destinationPath, boolean deleteSource) {
  LOG.debug(
      "Copying object '{}' to Object '{}'",
      String.format(BLOB_PATH_TEMPLATE, sourceBlobId.getBucket(), sourceBlobId.getName()),
      String.format(BLOB_PATH_TEMPLATE, destinationBucket, destinationPath)
  );

  Storage.CopyRequest copyRequest = new Storage.CopyRequest.Builder()
      .setSource(sourceBlobId)
      .setTarget(BlobId.of(destinationBucket, destinationPath))
      .build();
  Blob destinationBlob = storage.copy(copyRequest).getResult();
  LOG.debug(
      "Copied object '{}' to Object '{}'",
      String.format(BLOB_PATH_TEMPLATE, sourceBlobId.getBucket(), sourceBlobId.getName()),
      String.format(BLOB_PATH_TEMPLATE, destinationBlob.getBlobId().getBucket(), destinationBlob.getBlobId().getName())
  );
  if (deleteSource) {
    delete(sourceBlobId);
  }
}
 
Example #13
Source File: AbstractGCSIT.java    From nifi with Apache License 2.0 6 votes vote down vote up
@AfterClass
public static void tearDown() {
    try {
        // Empty the bucket before deleting it.
        Iterable<Blob> blobIterable = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll();

        for (final Blob blob : blobIterable) {
            storage.delete(blob.getBlobId());
        }

        storage.delete(BUCKET);
    } catch (final StorageException e) {
        fail("Unable to delete bucket " + BUCKET + ": " + e.getLocalizedMessage());
    }

    if (storage.get(BUCKET) != null) {
        fail("Incomplete teardown, subsequent tests might fail");
    }
}
 
Example #14
Source File: PutGCSObjectTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testAclAttributeGroup() throws Exception {
    reset(storage, blob);
    final PutGCSObject processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.create(any(BlobInfo.class), any(InputStream.class), any(Storage.BlobWriteOption.class)))
            .thenReturn(blob);

    final Acl.Group mockGroup = mock(Acl.Group.class);
    when(mockGroup.getEmail()).thenReturn(OWNER_GROUP_EMAIL);
    when(blob.getOwner()).thenReturn(mockGroup);

    runner.enqueue("test");
    runner.run();

    runner.assertAllFlowFilesTransferred(PutGCSObject.REL_SUCCESS);
    runner.assertTransferCount(PutGCSObject.REL_SUCCESS, 1);

    final MockFlowFile mockFlowFile = runner.getFlowFilesForRelationship(PutGCSObject.REL_SUCCESS).get(0);
    mockFlowFile.assertAttributeEquals(OWNER_ATTR, OWNER_GROUP_EMAIL);
    mockFlowFile.assertAttributeEquals(OWNER_TYPE_ATTR, "group");
}
 
Example #15
Source File: GcsSinkTaskGroupByTopicPartitionPropertiesTest.java    From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0 6 votes vote down vote up
private void genericTry(final List<List<SinkRecord>> recordBatches,
                        final Integer maxRecordsPerFile) {
    final Storage storage = LocalStorageHelper.getOptions().getService();
    final BucketAccessor testBucketAccessor = new BucketAccessor(storage, TEST_BUCKET, true);

    final Map<String, String> taskProps = basicTaskProps();
    if (maxRecordsPerFile != null) {
        taskProps.put(GcsSinkConfig.FILE_MAX_RECORDS, Integer.toString(maxRecordsPerFile));
    }
    final GcsSinkTask task = new GcsSinkTask(taskProps, storage);

    for (final List<SinkRecord> recordBatch : recordBatches) {
        task.put(recordBatch);
        task.flush(null);
    }

    checkExpectedFileNames(recordBatches, maxRecordsPerFile, testBucketAccessor);
    checkFileSizes(testBucketAccessor, maxRecordsPerFile);
    final int expectedRecordCount = recordBatches.stream().mapToInt(List::size).sum();
    checkTotalRecordCountAndNoMultipleWrites(expectedRecordCount, testBucketAccessor);
    checkTopicPartitionPartInFileNames(testBucketAccessor);
    checkOffsetOrderInFiles(testBucketAccessor);
}
 
Example #16
Source File: ChangeDefaultStorageClass.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
public static void changeDefaultStorageClass(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // See the StorageClass documentation for other valid storage classes:
  // https://googleapis.dev/java/google-cloud-clients/latest/com/google/cloud/storage/StorageClass.html
  StorageClass storageClass = StorageClass.COLDLINE;

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();
  Bucket bucket = storage.get(bucketName);
  bucket = bucket.toBuilder().setStorageClass(storageClass).build().update();

  System.out.println(
      "Default storage class for bucket "
          + bucketName
          + " has been set to "
          + bucket.getStorageClass());
}
 
Example #17
Source File: AbstractGCSIT.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@AfterClass
public static void tearDown() {
    try {
        // Empty the bucket before deleting it.
        Iterator<Blob> blobIterator = storage.list(BUCKET, Storage.BlobListOption.versions(true)).iterateAll();

        while(blobIterator.hasNext()) {
            Blob blob = blobIterator.next();
            storage.delete(blob.getBlobId());
        }

        storage.delete(BUCKET);
    } catch (final StorageException e) {
        fail("Unable to delete bucket " + BUCKET + ": " + e.getLocalizedMessage());
    }

    if (storage.get(BUCKET) != null) {
        fail("Incomplete teardown, subsequent tests might fail");
    }
}
 
Example #18
Source File: DownloadEncryptedObject.java    From google-cloud-java with Apache License 2.0 6 votes vote down vote up
public static void downloadEncryptedObject(
    String projectId,
    String bucketName,
    String objectName,
    Path destFilePath,
    String decryptionKey)
    throws IOException {

  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // The ID of your GCS object
  // String objectName = "your-object-name";

  // The path to which the file should be downloaded
  // Path destFilePath = Paths.get("/local/path/to/file.txt");

  // The Base64 encoded decryption key, which should be the same key originally used to encrypt
  // the object
  // String decryptionKey = "TIbv/fjexq+VmtXzAlc63J4z5kFmWJ6NdAPQulQBT7g=";

  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  Blob blob = storage.get(bucketName, objectName);
  blob.downloadTo(destFilePath, Blob.BlobSourceOption.decryptionKey(decryptionKey));

  System.out.println(
      "Downloaded object "
          + objectName
          + " from bucket name "
          + bucketName
          + " to "
          + destFilePath
          + " using customer-supplied encryption key");
}
 
Example #19
Source File: ListGCSBucketTest.java    From nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testListOptionsVersions() throws Exception {
    reset(storage, mockBlobPage);
    final ListGCSBucket processor = getProcessor();
    final TestRunner runner = buildNewRunner(processor);
    addRequiredPropertiesToRunner(runner);

    runner.setProperty(ListGCSBucket.USE_GENERATIONS, String.valueOf(USE_GENERATIONS));
    runner.assertValid();

    final Iterable<Blob> mockList = ImmutableList.of();

    when(mockBlobPage.getValues()).thenReturn(mockList);
    when(mockBlobPage.getNextPage()).thenReturn(null);
    when(storage.list(anyString(), argumentCaptor.capture())).thenReturn(mockBlobPage);

    runner.enqueue("test");
    runner.run();

    Storage.BlobListOption option = argumentCaptor.getValue();
    assertEquals(Storage.BlobListOption.versions(true), option);
}
 
Example #20
Source File: FetchGCSObjectTest.java    From localization_nifi with Apache License 2.0 6 votes vote down vote up
@Test
public void testStorageExceptionOnFetch() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    when(storage.get(any(BlobId.class))).thenThrow(new StorageException(400, "test-exception"));
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    runner.assertAllFlowFilesTransferred(FetchGCSObject.REL_FAILURE);
    runner.assertTransferCount(FetchGCSObject.REL_FAILURE, 1);
}
 
Example #21
Source File: ImportProductSetsIT.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
@After
public void tearDown() throws IOException {
  ProductManagement.deleteProduct(PROJECT_ID, COMPUTE_REGION, PRODUCT_ID_1);
  ProductSetManagement.deleteProductSet(PROJECT_ID, COMPUTE_REGION, PRODUCT_SET_ID);
  Storage storage = StorageOptions.newBuilder().setProjectId(PROJECT_ID).build().getService();
  // Delete the created blob
  storage.delete(blob.getBlobId());
  System.setOut(null);
}
 
Example #22
Source File: RemoveBucketIamMember.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
public static void removeBucketIamMember(String projectId, String bucketName) {
  // The ID of your GCP project
  // String projectId = "your-project-id";

  // The ID of your GCS bucket
  // String bucketName = "your-unique-bucket-name";

  // For more information please read:
  // https://cloud.google.com/storage/docs/access-control/iam
  Storage storage = StorageOptions.newBuilder().setProjectId(projectId).build().getService();

  Policy originalPolicy =
      storage.getIamPolicy(bucketName, Storage.BucketSourceOption.requestedPolicyVersion(3));

  String role = "roles/storage.objectViewer";
  String member = "group:[email protected]";

  // getBindingsList() returns an ImmutableList and copying over to an ArrayList so it's mutable.
  List<Binding> bindings = new ArrayList(originalPolicy.getBindingsList());

  // Remove role-member binding without a condition.
  for (int index = 0; index < bindings.size(); index++) {
    Binding binding = bindings.get(index);
    boolean foundRole = binding.getRole().equals(role);
    boolean foundMember = binding.getMembers().contains(member);
    boolean bindingIsNotConditional = binding.getCondition() == null;

    if (foundRole && foundMember && bindingIsNotConditional) {
      bindings.set(index, binding.toBuilder().removeMembers(member).build());
      break;
    }
  }

  // Update policy to remove member
  Policy.Builder updatedPolicyBuilder = originalPolicy.toBuilder();
  updatedPolicyBuilder.setBindings(bindings).setVersion(3);
  Policy updatedPolicy = storage.setIamPolicy(bucketName, updatedPolicyBuilder.build());

  System.out.printf("Removed %s with role %s from %s\n", member, role, bucketName);
}
 
Example #23
Source File: Gcs.java    From gcp-ingestion with Mozilla Public License 2.0 5 votes vote down vote up
private Write(Storage storage, long maxBytes, int maxMessages, Duration maxDelay,
    PubsubMessageToTemplatedString batchKeyTemplate, Executor executor,
    Function<Blob, CompletableFuture<Void>> batchCloseHook) {
  super(maxBytes, maxMessages, maxDelay, batchKeyTemplate, executor);
  this.storage = storage;
  this.batchCloseHook = batchCloseHook;
}
 
Example #24
Source File: GcsRepository.java    From hawkbit-extensions with Eclipse Public License 1.0 5 votes vote down vote up
@Override
public void deleteByTenant(final String tenant) {
    final String folder = sanitizeTenant(tenant);

    LOG.info("Deleting GCS object folder (tenant) from bucket {} and key {}", gcsProperties.getBucketName(),
            folder);
    final Page<Blob> blobs = gcsStorage.list(gcsProperties.getBucketName(),
            Storage.BlobListOption.currentDirectory(), Storage.BlobListOption.prefix(tenant));
    for (final Blob blob : blobs.iterateAll()) {
        gcsStorage.delete(blob.getBlobId());
    }
}
 
Example #25
Source File: ITStorageHmacKeySnippets.java    From google-cloud-java with Apache License 2.0 5 votes vote down vote up
private static void cleanUpHmacKeys(ServiceAccount serviceAccount) {
  Page<HmacKeyMetadata> page =
      storage.listHmacKeys(Storage.ListHmacKeysOption.serviceAccount(serviceAccount));
  for (HmacKeyMetadata metadata : page.iterateAll()) {
    if (metadata.getState() == HmacKeyState.ACTIVE) {
      storage.updateHmacKeyState(metadata, HmacKeyState.INACTIVE);
    }
    storage.deleteHmacKey(metadata);
  }
}
 
Example #26
Source File: AuthExample.java    From java-docs-samples with Apache License 2.0 5 votes vote down vote up
static void authAppEngineStandard() throws IOException {
  // Explicitly request service account credentials from the app engine standard instance.
  GoogleCredentials credentials = AppEngineCredentials.getApplicationDefault();
  Storage storage = StorageOptions.newBuilder().setCredentials(credentials).build().getService();

  System.out.println("Buckets:");
  Page<Bucket> buckets = storage.list();
  for (Bucket bucket : buckets.iterateAll()) {
    System.out.println(bucket.toString());
  }
}
 
Example #27
Source File: GcsPluginUtilsTest.java    From nomulus with Apache License 2.0 5 votes vote down vote up
@Test
public void testUploadFileToGcs() {
  Storage storage = mock(Storage.class);
  uploadFileToGcs(
      storage, "my-bucket", Paths.get("my", "filename.txt"), toByteArraySupplier("my data"));
  verify(storage)
      .create(
          BlobInfo.newBuilder("my-bucket", "my/filename.txt")
              .setContentType("text/plain")
              .build(),
          "my data".getBytes(UTF_8));
  verifyNoMoreInteractions(storage);
}
 
Example #28
Source File: GcsArtifact.java    From hawkbit-extensions with Eclipse Public License 1.0 5 votes vote down vote up
GcsArtifact(final Storage gcsStorage, final GcsRepositoryProperties gcsProperties, final String key,
        final String artifactId, final DbArtifactHash hashes, final Long size, final String contentType) {
    super(artifactId, hashes, size, contentType);
    Assert.notNull(gcsStorage, "GCS cannot be null");
    Assert.notNull(gcsProperties, "Properties cannot be null");
    Assert.notNull(key, "Key cannot be null");
    this.gcsStorage = gcsStorage;
    this.gcsProperties = gcsProperties;
    this.key = key;
}
 
Example #29
Source File: PutGCSObjectTest.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public PutGCSObject getProcessor() {
    return new PutGCSObject() {
        @Override
        protected Storage getCloudService() {
            return storage;
        }
    };
}
 
Example #30
Source File: ReportUploader.java    From nomulus with Apache License 2.0 5 votes vote down vote up
private void uploadResultsToGcs(String destination) {
  checkArgument(
      !destination.isEmpty(), "destination must include at least the bucket name, but is empty");
  Path bucketWithFolder = Paths.get(destination, createUniqueFolderName());
  String bucket = bucketWithFolder.getName(0).toString();
  Path folder = bucketWithFolder.subpath(1, bucketWithFolder.getNameCount());

  StorageOptions.Builder storageOptions = StorageOptions.newBuilder();
  if (!isNullOrEmpty(credentialsFile)) {
    try {
      storageOptions.setCredentials(
          GoogleCredentials.fromStream(new FileInputStream(credentialsFile)));
    } catch (IOException e) {
      throw new UncheckedIOException(e);
    }
  }
  Storage storage = storageOptions.build().getService();

  FilesWithEntryPoint filesToUpload = generateFilesToUpload();

  System.out.format(
      "ReportUploader: going to upload %s files to %s/%s\n",
      filesToUpload.files().size(), bucket, folder);
  if ("yes".equals(multithreadedUpload)) {
    System.out.format("ReportUploader: multi-threaded upload\n");
    uploadFilesToGcsMultithread(storage, bucket, folder, filesToUpload.files());
  } else {
    System.out.format("ReportUploader: single threaded upload\n");
    filesToUpload
        .files()
        .forEach(
            (path, dataSupplier) -> {
              System.out.format("ReportUploader: Uploading %s\n", path);
              uploadFileToGcs(storage, bucket, folder.resolve(path), dataSupplier);
            });
  }
  System.out.format(
      "ReportUploader: report uploaded to https://storage.googleapis.com/%s/%s\n",
      bucket, folder.resolve(filesToUpload.entryPoint()));
}