org.jclouds.io.Payload Java Examples

/**
 * Upload an object from a File using the Swift API.
 */
private void uploadObjectFromFile() throws IOException {
   System.out.format("Upload Object From File%n");

   String filename = "uploadObjectFromFile";
   String suffix = ".txt";

   File tempFile = File.createTempFile(filename, suffix);

   try {
      Files.write("uploadObjectFromFile", tempFile, Charsets.UTF_8);

      ByteSource byteSource = Files.asByteSource(tempFile);
      Payload payload = Payloads.newByteSourcePayload(byteSource);

      cloudFiles.getObjectApi(REGION, CONTAINER)
         .put(filename + suffix, payload);

      System.out.format("  %s%s%n", filename, suffix);
   } finally {
      tempFile.delete();
   }
}
 

@Test
public void testReadingFullObject() throws Exception {
    String objectKey = "testReadingFull";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);
    RandomInputStream toCompare = new RandomInputStream(0, objectSize);

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);
    assertStreamsMatch(toTest, toCompare);
}
 

@Test
public void testReadingFullObjectByBytes() throws Exception {
    String objectKey = "testReadingFull2";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);
    RandomInputStream toCompare = new RandomInputStream(0, objectSize);

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);
    assertStreamsMatchByBytes(toTest, toCompare);
}
 

private void generateGetTempURL() throws IOException {
   System.out.format("Generate GET Temp URL%n");

   HttpRequest request = blobStoreContext.getSigner(REGION).signGetBlob(CONTAINER, FILENAME, TEN_MINUTES);

   System.out.format("  %s %s%n", request.getMethod(), request.getEndpoint());

   // GET the file using jclouds
   File file = File.createTempFile(FILENAME, ".tmp");
   Payload payload = blobStoreContext.utils().http().invoke(request).getPayload();

   try {
      Files.asByteSink(file).writeFrom(payload.openStream());

      System.out.format("  GET Success (%s)%n", file.getAbsolutePath());
   } finally {
      payload.release();
      file.delete();
   }
}
 

private void generatePutTempURL() throws IOException {
   System.out.format("Generate PUT Temp URL%n");

   // Create the Payload
   String data = "This object will be public for 10 minutes.";
   ByteSource source = ByteSource.wrap(data.getBytes());
   Payload payload = Payloads.newByteSourcePayload(source);

   // Create the Blob
   Blob blob = blobStore.blobBuilder(FILENAME).payload(payload).contentType("text/plain").build();
   HttpRequest request = blobStoreContext.getSigner(REGION).signPutBlob(CONTAINER, blob, TEN_MINUTES);

   System.out.format("  %s %s%n", request.getMethod(), request.getEndpoint());

   // PUT the file using jclouds
   HttpResponse response = blobStoreContext.utils().http().invoke(request);
   int statusCode = response.getStatusCode();

   if (statusCode >= 200 && statusCode < 299) {
      System.out.format("  PUT Success (%s)%n", statusCode);
   }
   else {
      throw new HttpResponseException(null, response);
   }
}
 

/**
 * This method will put a plain text object into the container.
 */
private void createObjectFromFile() throws IOException {
   System.out.format("Create Object From File%n");

   File tempFile = File.createTempFile(FILENAME, SUFFIX);

   try {
      Files.write("Hello Cloud Files", tempFile, Charsets.UTF_8);

      ObjectApi objectApi = cloudFiles.getObjectApi(REGION, CONTAINER_PUBLISH);

      ByteSource byteSource = Files.asByteSource(tempFile);
      Payload payload = Payloads.newByteSourcePayload(byteSource);

      objectApi.put(FILENAME + SUFFIX, payload);
   } finally {
      tempFile.delete();
   }
}
 

@Override
public Reader get( EntityReference entityReference )
    throws EntityStoreException
{
    Blob blob = storeContext.getBlobStore().getBlob( container, entityReference.identity().toString() );
    if( blob == null )
    {
        throw new EntityNotFoundException( entityReference );
    }
    Payload payload = blob.getPayload();
    if( payload == null )
    {
        throw new EntityNotFoundException( entityReference );
    }
    try( InputStream input = payload.openStream() )
    {
        String state = new Scanner( input, StandardCharsets.UTF_8.name() ).useDelimiter( "\\Z" ).next();
        return new StringReader( state );
    }
    catch( IOException ex )
    {
        throw new EntityStoreException( "Unable to read entity state for: " + entityReference, ex );
    }
}
 

@Override
public Stream<Reader> entityStates()
{
    return storeContext
        .getBlobStore().list( container ).stream()
        .map( metadata ->
              {
                  Payload payload = storeContext.getBlobStore()
                                                .getBlob( container, metadata.getName() )
                                                .getPayload();
                  if( payload == null )
                  {
                      EntityReference reference = EntityReference.parseEntityReference( metadata.getName() );
                      throw new EntityNotFoundException( reference );
                  }
                  try( InputStream input = payload.openStream() )
                  {
                      String state = new Scanner( input, UTF_8.name() ).useDelimiter( "\\Z" ).next();
                      return (Reader) new StringReader( state );
                  }
                  catch( IOException ex )
                  {
                      throw new EntityStoreException( ex );
                  }
              } );
}
 

/**
 * Upload a large object from a File using the BlobStore API.
 *
 * @throws ExecutionException
 * @throws InterruptedException
 */
private void uploadLargeObjectFromFile(File largeFile) throws InterruptedException, ExecutionException {
   System.out.format("Upload Large Object From File%n");

   ByteSource source = Files.asByteSource(largeFile);
   // create the payload and set the content length
   Payload payload = Payloads.newByteSourcePayload(source);
   payload.getContentMetadata().setContentLength(largeFile.length());

   Blob blob = blobStore.blobBuilder(largeFile.getName())
         .payload(payload)
         .build();

   // configure the blobstore to use multipart uploading of the file
   String eTag = blobStore.putBlob(CONTAINER, blob, multipart());

   System.out.format("  Uploaded %s eTag=%s", largeFile.getName(), eTag);
}
 

/**
 * Upload an object from a File using the Swift API.
 */
private void uploadObjectFromFile() throws IOException {
   System.out.format("Upload Object From File%n");

   String filename = "uploadObjectFromFile";
   String suffix = ".txt";

   File tempFile = File.createTempFile(filename, suffix);

   try {
      Files.write("uploadObjectFromFile", tempFile, Charsets.UTF_8);

      ByteSource byteSource = Files.asByteSource(tempFile);
      Payload payload = Payloads.newByteSourcePayload(byteSource);

      cloudFiles.getObjectApi(REGION, CONTAINER)
         .put(filename + suffix, payload);

      System.out.format("  %s%s%n", filename, suffix);
   } finally {
      tempFile.delete();
   }
}
 

public int readAfterOverwrite() throws IOException {
    int count = 0;
    for (int i = 0; i < iterations; ++i) {
        String blobName = makeBlobName();
        blobStore.putBlob(containerName, makeBlob(blobName, payload1));
        blobStore.putBlob(containerName, makeBlob(blobName, payload2));
        Blob getBlob = blobStoreRead.getBlob(containerName, blobName);
        if (getBlob == null) {
            ++count;
            continue;
        }
        try (Payload payload = getBlob.getPayload();
             InputStream is = payload.openStream()) {
            if (Arrays.equals(payload1.read(), ByteStreams.toByteArray(
                    is))) {
                ++count;
            }
        }
        blobStore.removeBlob(containerName, blobName);
    }
    return count;
}
 

public int readAfterDelete() throws IOException {
    int count = 0;
    for (int i = 0; i < iterations; ++i) {
        String blobName = makeBlobName();
        blobStoreRead.putBlob(containerName, makeBlob(blobName, payload1));
        blobStore.removeBlob(containerName, blobName);
        Blob getBlob = blobStoreRead.getBlob(containerName, blobName);
        if (getBlob != null) {
            ++count;
            try (Payload payload = getBlob.getPayload();
                 InputStream is = payload.openStream()) {
                ByteStreams.copy(is, ByteStreams.nullOutputStream());
            }
        }
    }
    return count;
}
 

public int readAfterCreate() throws IOException {
    int count = 0;
    for (int i = 0; i < iterations; ++i) {
        String blobName = makeBlobName();
        blobStore.putBlob(containerName, makeBlob(blobName, payload1));
        Blob getBlob = blobStoreRead.getBlob(containerName, blobName);
        if (getBlob == null) {
            ++count;
        } else {
            try (Payload payload = getBlob.getPayload();
                 InputStream is = payload.openStream()) {
                ByteStreams.copy(is, ByteStreams.nullOutputStream());
            }
        }
        blobStore.removeBlob(containerName, blobName);
    }
    return count;
}
 

@Override
public <T> T processFileContent(String space, String id, FileContentProcessor<T> fileContentProcessor) throws FileStorageException {
    FileEntry fileEntry = createFileEntry(space, id);
    try {
        Blob blob = getBlobWithRetries(fileEntry, 3);
        if (blob == null) {
            throw new FileStorageException(MessageFormat.format(Messages.FILE_WITH_ID_AND_SPACE_DOES_NOT_EXIST, fileEntry.getId(),
                                                                fileEntry.getSpace()));
        }
        Payload payload = blob.getPayload();
        return processContent(fileContentProcessor, payload);
    } catch (Exception e) {
        throw new FileStorageException(e);
    }
}
 

private static void providerExample(BlobStoreContext context) throws IOException {
   // Get the provider API
   GlacierClient client = context.unwrapApi(GlacierClient.class);

   // Create a vault
   final String vaultName =  "jclouds_providerExample_" + UUID.randomUUID().toString();
   client.createVault(vaultName);

   // Upload an archive
   Payload payload = new ByteSourcePayload(buildData(16));
   payload.getContentMetadata().setContentType(PLAIN_TEXT_UTF_8.toString());
   payload.getContentMetadata().setContentLength(16L);
   String archiveId = client.uploadArchive(vaultName, payload);

   // Create an archive retrieval job request
   JobRequest archiveRetrievalJobRequest = ArchiveRetrievalJobRequest.builder()
         .archiveId(archiveId)
         .description("retrieval job")
         .build();

   // Initiate job
   String jobId = client.initiateJob(vaultName, archiveRetrievalJobRequest);
   try {
      // Poll until the job is done
      new BasePollingStrategy(client).waitForSuccess(vaultName, jobId);

      // Get the job output
      Payload result = client.getJobOutput(vaultName, jobId);

      // Print the result
      System.out.println("The retrieved payload is: " + Strings2.toStringAndClose(result.openStream()));
   } catch (InterruptedException e) {
      Throwables.propagate(e);
   }
}
 

/**
 * {@inheritDoc}
 */
@Override
    public long saveInputStream(String id, String root, String filePath, InputStream stream) throws IOException {
        if(stream == null){
            return 0L;
        }
        ContainerAndName can = getContainerAndName(id, root, filePath);
        createContainerIfNotExist(can.container);

        InputStream in = markableInputStream(stream);
        long size = markableStreamLength(in);

        Payload payload = Payloads.newInputStreamPayload(in);

        try {
            BlobStore store = getBlobStore();
            String asciiID = Base64.encodeBase64String(id.getBytes("UTF8"));

            Blob blob = store.blobBuilder(can.name)
                .payload(payload)
                .contentLength(size)
                .userMetadata(ImmutableMap.of("id", asciiID, "path", filePath))
                .build();
            store.putBlob(can.container, blob);
        } finally {
            payload.release();
            Closeables.close(stream, true);
            Closeables.close(in, true);
        }

        return size;
    }
 

@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
        int partNumber, Payload payload) {
    long length;
    try (InputStream is = payload.openStream()) {
        length = ByteStreams.copy(is, ByteStreams.nullOutputStream());
    } catch (IOException ioe) {
        throw new RuntimeException(ioe);
    }

    byte[] array = Longs.toByteArray(length);
    ByteSourcePayload newPayload = new ByteSourcePayload(
            ByteSource.wrap(array));
    newPayload.setContentMetadata(payload.getContentMetadata());
    newPayload.getContentMetadata().setContentLength((long) array.length);
    newPayload.getContentMetadata().setContentMD5((HashCode) null);

    // create a single-part object which contains the logical length which
    // list and complete will read later
    Blob blob = blobBuilder(mpu.id() + "-" + partNumber)
            .payload(newPayload)
            .build();
    super.putBlob(mpu.containerName(), blob);

    MultipartPart part = super.uploadMultipartPart(mpu, partNumber,
            newPayload);
    return MultipartPart.create(part.partNumber(), length, part.partETag(),
            part.lastModified());
}
 

@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
        int partNumber, Payload payload) {
    MultipartPart part = delegate().uploadMultipartPart(mpu, partNumber,
            payload);
    return part;
}
 

/**
 * {@inheritDoc}
 */
@Override
    public long saveInputStream(String id, String root, String filePath, InputStream stream) throws IOException {
        if(stream == null){
            return 0L;
        }
        ContainerAndName can = getContainerAndName(id, root, filePath);
        createContainerIfNotExist(can.container);

        InputStream in = markableInputStream(stream);
        long size = markableStreamLength(in);

        Payload payload = Payloads.newInputStreamPayload(in);

        try {
            BlobStore store = getBlobStore();
            String asciiID = Base64.encodeBase64String(id.getBytes("UTF8"));

            Blob blob = store.blobBuilder(can.name)
                .payload(payload)
                .contentLength(size)
                .userMetadata(ImmutableMap.of("id", asciiID, "path", filePath))
                .build();
            store.putBlob(can.container, blob);
        } finally {
            payload.release();
            Closeables.close(stream, true);
            Closeables.close(in, true);
        }

        return size;
    }
 

@Test
public void testSeekForward() throws Exception {
    String objectKey = "testSeekForward";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);

    // seek forward to middle
    long middle = objectSize/2;
    toTest.seekForward(middle);

    try {
        long before = middle - objectSize/4;
        toTest.seekForward(before);
        Assert.fail("Shound't be able to seek backwards");
    } catch (IOException ioe) {
        // correct
    }

    long after = middle + objectSize/4;
    RandomInputStream toCompare = new RandomInputStream(0, objectSize);
    toCompare.skip(after);

    toTest.seekForward(after);
    assertStreamsMatch(toTest, toCompare);
}
 

@Test
public void testSeek() throws Exception {
    String objectKey = "testSeek";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);

    Map<Integer, InputStream> seeks = new HashMap<>();
    Random r = new Random(12345);
    for (int i = 0; i < 20; i++) {
        int seek = r.nextInt(objectSize+1);
        RandomInputStream stream = new RandomInputStream(0, objectSize);
        stream.skip(seek);
        seeks.put(seek, stream);
    }

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(blobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);
    for (Map.Entry<Integer, InputStream> e : seeks.entrySet()) {
        toTest.seek(e.getKey());
        assertStreamsMatch(toTest, e.getValue());
    }
}
 

private <T> T processContent(FileContentProcessor<T> fileContentProcessor, Payload payload) throws FileStorageException {
    try (InputStream fileContentStream = payload.openStream()) {
        return fileContentProcessor.process(fileContentStream);
    } catch (Exception e) {
        throw new FileStorageException(e);
    }
}
 

@Test
public void testCreateMultipartBlobGetBlob() throws Exception {
    String blobName = "multipart-upload";
    BlobMetadata blobMetadata = makeBlob(nullBlobStore, blobName)
            .getMetadata();
    MultipartUpload mpu = nullBlobStore.initiateMultipartUpload(
            containerName, blobMetadata, new PutOptions());

    ByteSource byteSource = TestUtils.randomByteSource().slice(
            0, nullBlobStore.getMinimumMultipartPartSize() + 1);
    ByteSource byteSource1 = byteSource.slice(
            0, nullBlobStore.getMinimumMultipartPartSize());
    ByteSource byteSource2 = byteSource.slice(
            nullBlobStore.getMinimumMultipartPartSize(), 1);
    Payload payload1 = Payloads.newByteSourcePayload(byteSource1);
    Payload payload2 = Payloads.newByteSourcePayload(byteSource2);
    payload1.getContentMetadata().setContentLength(byteSource1.size());
    payload2.getContentMetadata().setContentLength(byteSource2.size());
    MultipartPart part1 = nullBlobStore.uploadMultipartPart(mpu, 1,
            payload1);
    MultipartPart part2 = nullBlobStore.uploadMultipartPart(mpu, 2,
            payload2);

    List<MultipartPart> parts = nullBlobStore.listMultipartUpload(mpu);
    assertThat(parts.get(0).partNumber()).isEqualTo(1);
    assertThat(parts.get(0).partSize()).isEqualTo(byteSource1.size());
    assertThat(parts.get(0).partETag()).isEqualTo(part1.partETag());
    assertThat(parts.get(1).partNumber()).isEqualTo(2);
    assertThat(parts.get(1).partSize()).isEqualTo(byteSource2.size());
    assertThat(parts.get(1).partETag()).isEqualTo(part2.partETag());

    assertThat(nullBlobStore.listMultipartUpload(mpu)).hasSize(2);

    nullBlobStore.completeMultipartUpload(mpu, parts);

    Blob newBlob = nullBlobStore.getBlob(containerName, blobName);
    validateBlobMetadata(newBlob.getMetadata());

    // content differs, only compare length
    try (InputStream actual = newBlob.getPayload().openStream();
            InputStream expected = byteSource.openStream()) {
        long actualLength = ByteStreams.copy(actual,
                ByteStreams.nullOutputStream());
        long expectedLength = ByteStreams.copy(expected,
                ByteStreams.nullOutputStream());
        assertThat(actualLength).isEqualTo(expectedLength);
    }

    nullBlobStore.removeBlob(containerName, blobName);
    assertThat(nullBlobStore.list(containerName)).isEmpty();
}
 

@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu,
        int partNumber, Payload payload) {
    throw new UnsupportedOperationException("read-only BlobStore");
}
 

@Test
public void testSeekWithinCurrent() throws Exception {
    String objectKey = "testSeekWithinCurrent";
    int objectSize = 12345;
    RandomInputStream toWrite = new RandomInputStream(0, objectSize);

    Payload payload = Payloads.newInputStreamPayload(toWrite);
    payload.getContentMetadata().setContentLength((long)objectSize);
    Blob blob = blobStore.blobBuilder(objectKey)
        .payload(payload)
        .contentLength((long)objectSize)
        .build();
    String ret = blobStore.putBlob(BUCKET, blob);
    log.debug("put blob: {} in Bucket: {}, in blobStore, result: {}", objectKey, BUCKET, ret);

    //BlobStore spiedBlobStore = spy(blobStore);
    BlobStore spiedBlobStore = mock(BlobStore.class, delegatesTo(blobStore));

    BackedInputStream toTest = new BlobStoreBackedInputStreamImpl(spiedBlobStore, BUCKET, objectKey,
                                                             (key, md) -> {},
                                                             objectSize, 1000);

    // seek forward
    RandomInputStream firstSeek = new RandomInputStream(0, objectSize);
    toTest.seek(100);
    firstSeek.skip(100);
    for (int i = 0; i < 100; i++) {
        Assert.assertEquals(firstSeek.read(), toTest.read());
    }

    // seek forward a bit more, but in same block
    RandomInputStream secondSeek = new RandomInputStream(0, objectSize);
    toTest.seek(600);
    secondSeek.skip(600);
    for (int i = 0; i < 100; i++) {
        Assert.assertEquals(secondSeek.read(), toTest.read());
    }

    // seek back
    RandomInputStream thirdSeek = new RandomInputStream(0, objectSize);
    toTest.seek(200);
    thirdSeek.skip(200);
    for (int i = 0; i < 100; i++) {
        Assert.assertEquals(thirdSeek.read(), toTest.read());
    }

    verify(spiedBlobStore, times(1))
        .getBlob(Mockito.eq(BUCKET), Mockito.eq(objectKey), any());
}
 

@Override
public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) {
    throw new UnsupportedOperationException();
}
 

/**
 * Upload an object from a String using the Swift API.
 */
private void uploadObjectFromString() {
   System.out.format("Upload Object From String%n");

   String filename = "uploadObjectFromString.txt";

   ByteSource source = ByteSource.wrap("uploadObjectFromString".getBytes());
   Payload payload = Payloads.newByteSourcePayload(source);

   cloudFiles.getObjectApi(REGION, CONTAINER).put(filename, payload);

   System.out.format("  %s%n", filename);
}
 

/**
 * Upload an object from a String using the Swift API.
 */
private void uploadObjectFromString() {
   System.out.format("Upload Object From String%n");

   String filename = "uploadObjectFromString.txt";

   ByteSource source = ByteSource.wrap("uploadObjectFromString".getBytes());
   Payload payload = Payloads.newByteSourcePayload(source);

   cloudFiles.getObjectApi(REGION, CONTAINER).put(filename, payload);

   System.out.format("  %s%n", filename);
}