com.google.cloud.storage.Storage.BlobListOption Java Examples

The following examples show how to use com.google.cloud.storage.Storage.BlobListOption. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ITTranslateSnippetsBeta.java From google-cloud-java with Apache License 2.0

6 votes

@Test
public void test1_testBatchTranslateText() {

  BatchTranslateResponse response =
      TranslateSnippetsBeta.batchTranslateText(
          projectId,
          "us-central1",
          "gs://cloud-samples-data/translation/text.txt",
          "gs://" + projectId + "/BATCH_TRANSLATION_OUTPUT/");
  assertEquals(13, response.getTotalCharacters());
  assertEquals(13, response.getTranslatedCharacters());
  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          projectId,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix("BATCH_TRANSLATION_OUTPUT/"));

  deleteDirectory(storage, blobs);
}

Example #2

Source File: GCSStorage.java From digdag with Apache License 2.0

6 votes

@Override
public void list(String objectPrefix, FileListing callback)
{
    checkArgument(objectPrefix != null, "objectPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + objectPrefix;
    Page<Blob> blobs = getWithRetry(errorMessage, () ->
            storage.list(bucket, BlobListOption.prefix(objectPrefix))
    );

    List<StorageObjectSummary> objectSummaryList = new ArrayList<>();
    for (Blob blob : blobs.iterateAll()) {
        objectSummaryList.add(
                StorageObjectSummary.builder()
                        .key(blob.getName())
                        .contentLength(blob.getSize())
                        .lastModified(convertToInstant(blob))
                        .build()
        );
    }
    callback.accept(objectSummaryList);
}

Example #3

Source File: DocumentOcrTemplate.java From spring-cloud-gcp with Apache License 2.0

6 votes

/**
 * Parses the OCR output files who have the specified {@code jsonFilesetPrefix}. This
 * method assumes that all of the OCR output files with the prefix are a part of the same
 * document.
 *
 * @param jsonOutputFilePathPrefix the folder location containing all of the JSON files of
 *     OCR output
 * @return A {@link DocumentOcrResultSet} describing the OCR content of a document
 */
public DocumentOcrResultSet readOcrOutputFileSet(GoogleStorageLocation jsonOutputFilePathPrefix) {
	String nonNullPrefix = (jsonOutputFilePathPrefix.getBlobName() == null)
			? ""
			: jsonOutputFilePathPrefix.getBlobName();

	Page<Blob> blobsInFolder = this.storage.list(
			jsonOutputFilePathPrefix.getBucketName(),
			BlobListOption.currentDirectory(),
			BlobListOption.prefix(nonNullPrefix));

	List<Blob> blobPages =
			StreamSupport.stream(blobsInFolder.getValues().spliterator(), false)
					.filter(blob -> blob.getContentType().equals("application/octet-stream"))
					.collect(Collectors.toList());

	return new DocumentOcrResultSet(blobPages);
}

Example #4

Source File: SnowflakeGCSClient.java From snowflake-jdbc with Apache License 2.0

6 votes

/**
 * listObjects gets all the objects in a path
 *
 * @param remoteStorageLocation bucket name
 * @param prefix                Path
 * @return
 * @throws StorageProviderException
 */
@Override
public StorageObjectSummaryCollection listObjects(String remoteStorageLocation,
                                                  String prefix) throws StorageProviderException
{
  try
  {
    Page<Blob> blobs = this.gcsClient.list(remoteStorageLocation, BlobListOption.prefix(prefix));
    return new StorageObjectSummaryCollection(blobs);
  }
  catch (Exception e)
  {
    logger.debug("Failed to list objects");
    throw new StorageProviderException(e);
  }
}

Example #5

Source File: StreamingAnnotationToStorageIT.java From java-docs-samples with Apache License 2.0

6 votes

@Test
public void testStreamingAnnotationToStorage() {
  String gcsUri = String.format("gs://%s/%s", PROJECT_ID, OUTPUT_PREFIX);
  StreamingAnnotationToStorage.streamingAnnotationToStorage("resources/cat.mp4", gcsUri);
  String got = bout.toString();

  assertThat(got).contains(String.format("Storage Uri: %s", gcsUri));

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          PROJECT_ID,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));

  deleteDirectory(storage, blobs);
}

Example #6

Source File: DetectIT.java From java-docs-samples with Apache License 2.0

6 votes

@Test
public void testDetectDocumentsGcs() throws Exception {
  // Act
  Detect.detectDocumentsGcs(
      "gs://" + ASSET_BUCKET + "/vision/document/custom_0773375000.pdf",
      "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/");

  // Assert
  String got = bout.toString();

  assertThat(got).contains("OIL, GAS AND MINERAL LEASE");

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs =
      storage.list(
          OUTPUT_BUCKET,
          BlobListOption.currentDirectory(),
          BlobListOption.prefix(OUTPUT_PREFIX + "/"));
  for (Blob blob : blobs.iterateAll()) {
    blob.delete();
  }
}

Example #7

Source File: DetectIT.java From java-docs-samples with Apache License 2.0

6 votes

@Test
public void testAsyncBatchAnnotateImagesGcs() throws Exception {
  // Act
  AsyncBatchAnnotateImagesGcs.asyncBatchAnnotateImagesGcs(
      "gs://cloud-samples-data/vision/label/wakeupcat.jpg",
      "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/");

  // Assert
  String got = bout.toString();
  assertThat(got).contains("red:");

  Storage storage = StorageOptions.getDefaultInstance().getService();

  Page<Blob> blobs = storage.list(OUTPUT_BUCKET, BlobListOption.currentDirectory(),
      BlobListOption.prefix(OUTPUT_PREFIX + "/"));
  for (Blob blob : blobs.iterateAll()) {
    blob.delete();
  }
}

Example #8

Source File: QuickStartIT.java From java-docs-samples with Apache License 2.0

6 votes

private static final void deleteObjects() {
  Storage storage = StorageOptions.getDefaultInstance().getService();
  for (BlobInfo info :
      storage
          .list(
              bucketName,
              BlobListOption.versions(true),
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(path + "/"))
          .getValues()) {
    storage.delete(info.getBlobId());
  }
}

Example #9

Source File: GoogleCloudBlobStoreMetricsStore.java From nexus-blobstore-google-cloud with Eclipse Public License 1.0

5 votes

protected Stream<GoogleCloudPropertiesFile> backingFiles() {
  if (bucket == null) {
    return Stream.empty();
  } else {
    return stream(bucket.list(BlobListOption.prefix(nodeAccess.getId())).iterateAll())
        .filter(b -> b.getName().endsWith(METRICS_FILENAME))
        .map(blob -> new GoogleCloudPropertiesFile(bucket, blob.getName()));
  }
}

Example #10

Source File: ITTranslateSnippetsBeta.java From google-cloud-java with Apache License 2.0

5 votes

private void deleteDirectory(Storage storage, Page<Blob> blobs) {
  for (Blob blob : blobs.iterateAll()) {
    System.out.println(blob.getBlobId());
    if (!blob.delete()) {
      Page<Blob> subBlobs =
          storage.list(
              projectId,
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(blob.getName()));

      deleteDirectory(storage, subBlobs);
    }
  }
}

Example #11

Source File: StreamingAnnotationToStorageIT.java From java-docs-samples with Apache License 2.0

5 votes

private void deleteDirectory(Storage storage, Page<Blob> blobs) {
  for (Blob blob : blobs.iterateAll()) {
    System.out.println(blob.getName());
    if (!blob.delete()) {
      Page<Blob> subBlobs =
          storage.list(
              PROJECT_ID,
              BlobListOption.currentDirectory(),
              BlobListOption.prefix(blob.getName()));

      deleteDirectory(storage, subBlobs);
    }
  }
}

Example #12

Source File: OldGCSNotebookRepo.java From zeppelin with Apache License 2.0

5 votes

@Override
public List<OldNoteInfo> list(AuthenticationInfo subject) throws IOException {
  try {
    List<OldNoteInfo> infos = new ArrayList<>();
    Iterable<Blob> blobsUnderDir;
    if (basePath.isPresent()) {
      blobsUnderDir = storage
        .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
        .iterateAll();
    } else {
      blobsUnderDir = storage
        .list(bucketName)
        .iterateAll();
    }
    for (Blob b : blobsUnderDir) {
      Matcher matcher = noteNamePattern.matcher(b.getName());
      if (matcher.matches()) {
        // Callers only use the id field, so do not fetch each note
        // This matches the implementation in FileSystemNoteRepo#list
        infos.add(new OldNoteInfo(matcher.group(1), "", null));
      }
    }
    return infos;
  } catch (StorageException se) {
    throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
  }
}

Example #13

Source File: GCSNotebookRepo.java From zeppelin with Apache License 2.0

5 votes

@Override
public Map<String, NoteInfo> list(AuthenticationInfo subject) throws IOException {
  try {
    Map<String, NoteInfo> infos = new HashMap<>();
    Iterable<Blob> blobsUnderDir;
    if (basePath.isPresent()) {
      blobsUnderDir = storage
        .list(bucketName, BlobListOption.prefix(this.basePath.get() + "/"))
        .iterateAll();
    } else {
      blobsUnderDir = storage
        .list(bucketName)
        .iterateAll();
    }
    for (Blob b : blobsUnderDir) {
      Matcher matcher = notePathPattern.matcher(b.getName());
      if (matcher.matches()) {
        // Callers only use the id field, so do not fetch each note
        // This matches the implementation in FileSystemNoteRepo#list
        String noteFileName = matcher.group(1);
        try {
          String noteId = getNoteId(noteFileName);
          String notePath = getNotePath("", noteFileName);
          infos.put(noteId, new NoteInfo(noteId, notePath));
        } catch (IOException e) {
          LOGGER.warn(e.getMessage());
        }
      }
    }
    return infos;
  } catch (StorageException se) {
    throw new IOException("Could not list GCS directory: " + se.getMessage(), se);
  }
}

Example #14

Source File: GCPRestorer.java From cassandra-backup with Apache License 2.0

5 votes

private Page<Blob> list(final String bucket, final String pathPrefix) {
    return storage.list(bucket, BlobListOption.prefix(request.storageLocation.clusterId
                                                          + "/" + request.storageLocation.datacenterId
                                                          + "/" + request.storageLocation.nodeId
                                                          + "/" + pathPrefix + "/"),
                        BlobListOption.currentDirectory());
}

Example #15

Source File: StorageIntegrationTest.java From gcp-ingestion with Mozilla Public License 2.0

4 votes

private void downloadOutputFiles(String prefix) {
  Page<Blob> blobs = storage.list(bucket, BlobListOption.prefix(prefix));
  for (Blob blob : blobs.iterateAll()) {
    blob.downloadTo(Paths.get(tempFolder.getRoot().getPath(), blob.getName()));
  }
}

Example #16

Source File: GoogleCloudBlobStore.java From nexus-blobstore-google-cloud with Eclipse Public License 1.0

4 votes

Stream<BlobInfo> blobStream(final String path) {
  return stream(bucket.list(BlobListOption.prefix(path)).iterateAll()).map(c -> c);
}

Example #17

Source File: ITBlobSnippets.java From google-cloud-java with Apache License 2.0

4 votes

@After
public void after() {
  for (BlobInfo info : storage.list(BUCKET, BlobListOption.versions(true)).getValues()) {
    storage.delete(info.getBlobId());
  }
}