Java Code Examples for com.google.cloud.storage.Storage#BlobSourceOption

The following examples show how to use com.google.cloud.storage.Storage#BlobSourceOption . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testBlobIdWithGeneration() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);

    runner.removeProperty(FetchGCSObject.KEY);
    runner.removeProperty(FetchGCSObject.BUCKET);

    runner.setProperty(FetchGCSObject.GENERATION, String.valueOf(GENERATION));
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("", ImmutableMap.of(
            BUCKET_ATTR, BUCKET,
            CoreAttributes.FILENAME.key(), KEY
    ));

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertEquals(
            GENERATION,
            blobId.getGeneration()
    );


    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());
    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.generationMatch()));
    assertEquals(
            1,
            blobSourceOptions.size()
    );

}
 
Example 2
@Test
public void testBlobIdWithEncryption() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());

    runner.setProperty(FetchGCSObject.ENCRYPTION_KEY, ENCRYPTION_SHA256);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertNull(blobId.getGeneration());

    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());

    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.decryptionKey(ENCRYPTION_SHA256)));
    assertEquals(
            1,
            blobSourceOptions.size()
    );
}
 
Example 3
Source Project: nifi   File: FetchGCSObject.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();

    final String bucketName = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final Long generation = context.getProperty(GENERATION).evaluateAttributeExpressions(flowFile).asLong();
    final String encryptionKey = context.getProperty(ENCRYPTION_KEY).evaluateAttributeExpressions(flowFile).getValue();

    final Storage storage = getCloudService();
    final BlobId blobId = BlobId.of(bucketName, key, generation);

    try {
        final List<Storage.BlobSourceOption> blobSourceOptions = new ArrayList<>(2);

        if (encryptionKey != null) {
            blobSourceOptions.add(Storage.BlobSourceOption.decryptionKey(encryptionKey));
        }

        if (generation != null) {
            blobSourceOptions.add(Storage.BlobSourceOption.generationMatch());
        }

        final Blob blob = storage.get(blobId);
        if (blob == null) {
            throw new StorageException(404, "Blob " + blobId + " not found");
        }

        final ReadChannel reader = storage.reader(blobId, blobSourceOptions.toArray(new Storage.BlobSourceOption[0]));
        flowFile = session.importFrom(Channels.newInputStream(reader), flowFile);

        final Map<String, String> attributes = StorageAttributes.createAttributes(blob);
        flowFile = session.putAllAttributes(flowFile, attributes);
    } catch (StorageException e) {
        getLogger().error("Failed to fetch GCS Object due to {}", new Object[] {e}, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    session.transfer(flowFile, REL_SUCCESS);

    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully retrieved GCS Object for {} in {} millis; routing to success", new Object[]{flowFile, millis});
    session.getProvenanceReporter().fetch(flowFile, "https://" + bucketName + ".storage.googleapis.com/" + key, millis);
}
 
Example 4
Source Project: nifi   File: FetchGCSObjectTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBlobIdWithGeneration() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);

    runner.removeProperty(FetchGCSObject.KEY);
    runner.removeProperty(FetchGCSObject.BUCKET);

    runner.setProperty(FetchGCSObject.GENERATION, String.valueOf(GENERATION));
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("", ImmutableMap.of(
            BUCKET_ATTR, BUCKET,
            CoreAttributes.FILENAME.key(), KEY
    ));

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertEquals(
            GENERATION,
            blobId.getGeneration()
    );


    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());
    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.generationMatch()));
    assertEquals(
            1,
            blobSourceOptions.size()
    );

}
 
Example 5
Source Project: nifi   File: FetchGCSObjectTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testBlobIdWithEncryption() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());

    runner.setProperty(FetchGCSObject.ENCRYPTION_KEY, ENCRYPTION_SHA256);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertNull(blobId.getGeneration());

    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());

    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.decryptionKey(ENCRYPTION_SHA256)));
    assertEquals(
            1,
            blobSourceOptions.size()
    );
}