Java Code Examples for com.google.cloud.storage.Storage#BlobSourceOption

The following examples show how to use com.google.cloud.storage.Storage#BlobSourceOption . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FetchGCSObjectTest.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testBlobIdWithGeneration() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);

    runner.removeProperty(FetchGCSObject.KEY);
    runner.removeProperty(FetchGCSObject.BUCKET);

    runner.setProperty(FetchGCSObject.GENERATION, String.valueOf(GENERATION));
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("", ImmutableMap.of(
            BUCKET_ATTR, BUCKET,
            CoreAttributes.FILENAME.key(), KEY
    ));

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertEquals(
            GENERATION,
            blobId.getGeneration()
    );


    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());
    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.generationMatch()));
    assertEquals(
            1,
            blobSourceOptions.size()
    );

}

Example 2

Source File: FetchGCSObjectTest.java From localization_nifi with Apache License 2.0

4 votes

@Test
public void testBlobIdWithEncryption() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());

    runner.setProperty(FetchGCSObject.ENCRYPTION_KEY, ENCRYPTION_SHA256);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertNull(blobId.getGeneration());

    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());

    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.decryptionKey(ENCRYPTION_SHA256)));
    assertEquals(
            1,
            blobSourceOptions.size()
    );
}

Example 3

Source File: FetchGCSObject.java From nifi with Apache License 2.0

4 votes

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }

    final long startNanos = System.nanoTime();

    final String bucketName = context.getProperty(BUCKET).evaluateAttributeExpressions(flowFile).getValue();
    final String key = context.getProperty(KEY).evaluateAttributeExpressions(flowFile).getValue();
    final Long generation = context.getProperty(GENERATION).evaluateAttributeExpressions(flowFile).asLong();
    final String encryptionKey = context.getProperty(ENCRYPTION_KEY).evaluateAttributeExpressions(flowFile).getValue();

    final Storage storage = getCloudService();
    final BlobId blobId = BlobId.of(bucketName, key, generation);

    try {
        final List<Storage.BlobSourceOption> blobSourceOptions = new ArrayList<>(2);

        if (encryptionKey != null) {
            blobSourceOptions.add(Storage.BlobSourceOption.decryptionKey(encryptionKey));
        }

        if (generation != null) {
            blobSourceOptions.add(Storage.BlobSourceOption.generationMatch());
        }

        final Blob blob = storage.get(blobId);
        if (blob == null) {
            throw new StorageException(404, "Blob " + blobId + " not found");
        }

        final ReadChannel reader = storage.reader(blobId, blobSourceOptions.toArray(new Storage.BlobSourceOption[0]));
        flowFile = session.importFrom(Channels.newInputStream(reader), flowFile);

        final Map<String, String> attributes = StorageAttributes.createAttributes(blob);
        flowFile = session.putAllAttributes(flowFile, attributes);
    } catch (StorageException e) {
        getLogger().error("Failed to fetch GCS Object due to {}", new Object[] {e}, e);
        flowFile = session.penalize(flowFile);
        session.transfer(flowFile, REL_FAILURE);
        return;
    }

    session.transfer(flowFile, REL_SUCCESS);

    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
    getLogger().info("Successfully retrieved GCS Object for {} in {} millis; routing to success", new Object[]{flowFile, millis});
    session.getProvenanceReporter().fetch(flowFile, "https://" + bucketName + ".storage.googleapis.com/" + key, millis);
}

Example 4

Source File: FetchGCSObjectTest.java From nifi with Apache License 2.0

4 votes

@Test
public void testBlobIdWithGeneration() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());
    addRequiredPropertiesToRunner(runner);

    runner.removeProperty(FetchGCSObject.KEY);
    runner.removeProperty(FetchGCSObject.BUCKET);

    runner.setProperty(FetchGCSObject.GENERATION, String.valueOf(GENERATION));
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("", ImmutableMap.of(
            BUCKET_ATTR, BUCKET,
            CoreAttributes.FILENAME.key(), KEY
    ));

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertEquals(
            GENERATION,
            blobId.getGeneration()
    );


    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());
    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.generationMatch()));
    assertEquals(
            1,
            blobSourceOptions.size()
    );

}

Example 5

Source File: FetchGCSObjectTest.java From nifi with Apache License 2.0

4 votes

@Test
public void testBlobIdWithEncryption() throws Exception {
    reset(storage);
    final TestRunner runner = buildNewRunner(getProcessor());

    runner.setProperty(FetchGCSObject.ENCRYPTION_KEY, ENCRYPTION_SHA256);
    addRequiredPropertiesToRunner(runner);
    runner.assertValid();

    final Blob blob = mock(Blob.class);
    when(storage.get(any(BlobId.class))).thenReturn(blob);
    when(storage.reader(any(BlobId.class), any(Storage.BlobSourceOption.class))).thenReturn(new MockReadChannel(CONTENT));

    runner.enqueue("");

    runner.run();

    ArgumentCaptor<BlobId> blobIdArgumentCaptor = ArgumentCaptor.forClass(BlobId.class);
    ArgumentCaptor<Storage.BlobSourceOption> blobSourceOptionArgumentCaptor = ArgumentCaptor.forClass(Storage.BlobSourceOption.class);
    verify(storage).get(blobIdArgumentCaptor.capture());
    verify(storage).reader(any(BlobId.class), blobSourceOptionArgumentCaptor.capture());

    final BlobId blobId = blobIdArgumentCaptor.getValue();

    assertEquals(
            BUCKET,
            blobId.getBucket()
    );

    assertEquals(
            KEY,
            blobId.getName()
    );

    assertNull(blobId.getGeneration());

    final Set<Storage.BlobSourceOption> blobSourceOptions = ImmutableSet.copyOf(blobSourceOptionArgumentCaptor.getAllValues());

    assertTrue(blobSourceOptions.contains(Storage.BlobSourceOption.decryptionKey(ENCRYPTION_SHA256)));
    assertEquals(
            1,
            blobSourceOptions.size()
    );
}