Java Code Examples for com.amazonaws.services.s3.model.S3Object#getObjectContent()

@Override
public InputStream read( UUID appId, Entity entity, long offset, long length ) throws Exception {

    S3Object object = getS3Client().getObject( bucketName, AssetUtils.buildAssetKey( appId, entity ) );

    byte data[] = null;

    if ( offset == 0 && length == FIVE_MB ) {
        return object.getObjectContent();
    }
    else {
        object.getObjectContent().read(data, Ints.checkedCast(offset), Ints.checkedCast(length));
    }

    return new ByteArrayInputStream(data);
}
 

/**
 * @return the new config file (datapipelines.properties) from s3 as a String
 */
private static String getNewConfig () {
  StringBuilder config = new StringBuilder();

  // get config object from S3
  AmazonS3Client s3Client = new AmazonS3Client();
  S3Object configObj = s3Client.getObject(DATAPIPELINES_CONFIG_S3_BUCKET,
      DATAPIPELINES_CONFIG_S3_KEY);

  // write object to String
  BufferedReader reader = new BufferedReader(new InputStreamReader(configObj.getObjectContent()));
  String line;
  try {
    while ((line = reader.readLine()) != null) {
      config.append(String.format("%s\n", line));
    }
  } catch (IOException e) {
    LOG.error("Failed to read config ({}) S3Object to String, exception: {}.",
        String.format("%s/%s", DATAPIPELINES_CONFIG_S3_BUCKET, DATAPIPELINES_CONFIG_S3_KEY),
        ExceptionUtils.getFullStackTrace(e));
  }

  return config.toString();
}
 

@Override
public void run() {
    SignedDomain signedDomain = null;
    try {
        S3Object object = s3.getObject(s3BucketName, domainName);
        try (S3ObjectInputStream s3is = object.getObjectContent()) {
            signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
        }
    } catch (Exception ex) {
        LOGGER.error("AWSS3ChangeLogThread: ObjectS3Thread- getSignedDomain - unable to get domain {} error: {}",
                domainName, ex.getMessage());
    }
    if (signedDomain != null) {
        signedDomainMap.put(domainName, signedDomain);
    }
}
 

@Test
public void testAtomicMpuAbort() throws Exception {
    String key = "testAtomicMpuAbort";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, key, BYTE_SOURCE.openStream(),
            metadata);

    InitiateMultipartUploadRequest initRequest =
            new InitiateMultipartUploadRequest(containerName, key);
    InitiateMultipartUploadResult initResponse =
            client.initiateMultipartUpload(initRequest);
    String uploadId = initResponse.getUploadId();

    client.abortMultipartUpload(new AbortMultipartUploadRequest(
                containerName, key, uploadId));

    S3Object object = client.getObject(containerName, key);
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 

@Override
public boolean load(BuildCacheKey key, BuildCacheEntryReader reader) {
  final String bucketPath = getBucketPath(key);
  if (s3.doesObjectExist(bucketName, bucketPath)) {
    logger.info("Found cache item '{}' in S3 bucket", bucketPath);
    S3Object object = s3.getObject(bucketName, bucketPath);
    try (InputStream is = object.getObjectContent()) {
      reader.readFrom(is);
      return true;
    } catch (IOException e) {
      throw new BuildCacheException("Error while reading cache object from S3 bucket", e);
    }
  } else {
    logger.info("Did not find cache item '{}' in S3 bucket", bucketPath);
    return false;
  }
}
 

@Test
public void testConditionalGet() throws Exception {
    assumeTrue(!blobStoreType.equals("b2"));

    String blobName = "blob-name";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    PutObjectResult result = client.putObject(containerName, blobName,
            BYTE_SOURCE.openStream(), metadata);

    S3Object object = client.getObject(
            new GetObjectRequest(containerName, blobName)
                    .withMatchingETagConstraint(result.getETag()));
    try (InputStream is = object.getObjectContent()) {
        assertThat(is).isNotNull();
        ByteStreams.copy(is, ByteStreams.nullOutputStream());
    }

    object = client.getObject(
            new GetObjectRequest(containerName, blobName)
                    .withNonmatchingETagConstraint(result.getETag()));
    assertThat(object).isNull();
}
 

/**
 * S3 block read would be achieved through the AmazonS3 client. Following
 * are the steps to achieve: (1) Create the objectRequest from bucketName
 * and filePath. (2) Set the range to the above created objectRequest. (3)
 * Get the object portion through AmazonS3 client API. (4) Get the object
 * content from the above object portion.
 *
 * @param bytesFromCurrentOffset
 *          bytes read till now from current offset
 * @param bytesToFetch
 *          the number of bytes to be fetched
 * @return the number of bytes read, -1 if 0 bytes read
 * @throws IOException
 */

@Override
protected int readData(final long bytesFromCurrentOffset, final int bytesToFetch) throws IOException
{
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(s3Params.bucketName, s3Params.filePath);
  rangeObjectRequest.setRange(offset + bytesFromCurrentOffset, offset + bytesFromCurrentOffset + bytesToFetch - 1);
  S3Object objectPortion = s3Params.s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  buffer = ByteStreams.toByteArray(wrappedStream);
  wrappedStream.close();
  int bufferLength = buffer.length;
  if (bufferLength <= 0) {
    return -1;
  }
  return bufferLength;
}
 

@Override
public InputStream readBlob(String blobName) throws IOException {
    try (AmazonS3Reference clientReference = blobStore.clientReference()) {
        final S3Object s3Object = clientReference.client().getObject(blobStore.bucket(), buildKey(blobName));
        return s3Object.getObjectContent();
    } catch (final AmazonClientException e) {
        if (e instanceof AmazonS3Exception) {
            if (404 == ((AmazonS3Exception) e).getStatusCode()) {
                throw new NoSuchFileException("Blob object [" + blobName + "] not found: " + e.getMessage());
            }
        }
        throw e;
    }
}
 

@Test
public void testCopyObject() throws Exception {
  String newName = UUID.randomUUID().toString();

  AmazonS3ExecutorConfig config = getConfig();
  config.taskConfig.taskType = TaskType.COPY_OBJECT;
  config.taskConfig.copyTargetLocation = newName;

  AmazonS3Executor executor = new AmazonS3Executor(config);
  TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
    .build();
  runner.runInit();

  try {
    s3client.putObject(new PutObjectRequest(BUCKET_NAME, objectName, IOUtils.toInputStream("content"), new ObjectMetadata()));
    runner.runWrite(ImmutableList.of(getTestRecord()));

    S3Object object = s3client.getObject(BUCKET_NAME, newName);
    S3ObjectInputStream objectContent = object.getObjectContent();

    List<String> stringList = IOUtils.readLines(objectContent);
    Assert.assertEquals(1, stringList.size());
    Assert.assertEquals("content", stringList.get(0));

    Assert.assertTrue(s3client.doesObjectExist(BUCKET_NAME, objectName));

    Assert.assertEquals(1, runner.getEventRecords().size());
    assertEvent(runner.getEventRecords().get(0), newName);
  } finally {
    runner.runDestroy();
  }
}
 

@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
    int parallelism = context.getNumberOfParallelSubtasks();
    if (parallelism != 1) {
        throw new IllegalStateException("SeedUrlSource only supports a parallelism of 1");
    }

    if (_terminator == null) {
        throw new IllegalStateException("Crawl terminator must be set for the seed URL source");
    }
    
    LOGGER.info("Opening seed URL source");

    // Open the terminator, so that it knows when we really started running.
    _terminator.open();
    
    _urlIndex = 0;
    
    if (useS3File()) {
        AmazonS3 s3Client = S3Utils.makeS3Client();
        S3Object object = s3Client
                .getObject(new GetObjectRequest(_seedUrlsS3Bucket, _seedUrlsS3Path));
        _s3FileStream = object.getObjectContent();
    }
}
 

public String getString(String bucket, String key) {
    GetObjectRequest request = new GetObjectRequest(bucket, key);
    S3Object response = client.getObject(request);
    try (S3ObjectInputStream is = response.getObjectContent()) {
        return CharStreams.toString(new InputStreamReader(is, Charsets.UTF_8));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}
 

/**
 * Constructor for the S3 version of the Bloom filter.
 * @param name String. The name of the object.
 * @param object S3Object. The object that contains the file.
 * @throws Exception on S3 errors.
 */
public Bloom(String name, S3Object object, long size) throws Exception {
	InputStream objectData = object.getObjectContent();
	BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
	makeFilter(br,size);
	
	symbols.put(name, bloomFilter);
}
 

@Test
public void testCopyObjectPreserveMetadata() throws Exception {
    String fromName = "from-name";
    String toName = "to-name";
    String cacheControl = "max-age=3600";
    String contentDisposition = "attachment; filename=old.jpg";
    String contentEncoding = "gzip";
    String contentLanguage = "en";
    String contentType = "audio/ogg";
    Map<String, String> userMetadata = ImmutableMap.of(
            "key1", "value1",
            "key2", "value2");
    ObjectMetadata metadata = new ObjectMetadata();
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        metadata.setCacheControl(cacheControl);
    }
    metadata.setContentLength(BYTE_SOURCE.size());
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        metadata.setContentDisposition(contentDisposition);
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        metadata.setContentEncoding(contentEncoding);
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        metadata.setContentLanguage(contentLanguage);
    }
    metadata.setContentType(contentType);
    // TODO: expires
    metadata.setUserMetadata(userMetadata);
    client.putObject(containerName, fromName, BYTE_SOURCE.openStream(),
            metadata);

    client.copyObject(containerName, fromName, containerName, toName);

    S3Object object = client.getObject(containerName, toName);

    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }

    ObjectMetadata contentMetadata = object.getObjectMetadata();
    assertThat(contentMetadata.getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        assertThat(contentMetadata.getCacheControl()).isEqualTo(
                cacheControl);
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        assertThat(contentMetadata.getContentDisposition()).isEqualTo(
                contentDisposition);
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        assertThat(contentMetadata.getContentEncoding()).isEqualTo(
                contentEncoding);
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        assertThat(contentMetadata.getContentLanguage()).isEqualTo(
                contentLanguage);
    }
    assertThat(contentMetadata.getContentType()).isEqualTo(
            contentType);
    // TODO: expires
    assertThat(contentMetadata.getUserMetadata()).isEqualTo(
            userMetadata);
}
 

/**
 * Fetchs a remote object from S3; datasource, bucket, key, aes256key supported
 * 
 * @param props
 * @param _Session
 * @throws cfmRunTimeException
 */
private void remoteFetchS3( cfStructData props, cfSession _Session ) throws cfmRunTimeException {

	if ( !props.containsKey( "datasource" ) ||
			!props.containsKey( "bucket" ) ||
			!props.containsKey( "key" ) )
		throw newRunTimeException( "'remote'.type=s3; minimum keys are datasource, bucket and key" );

	String datasource = props.getData( "datasource" ).getString();
	String bucket = props.getData( "bucket" ).getString();
	String key = props.getData( "key" ).getString();

	// Get the Amazon datasource
	AmazonKey amazonKey = AmazonKeyFactory.getDS( datasource );
	if ( amazonKey == null )
		throw newRunTimeException( "Amazon Datasource [" + datasource + "] has not been registered; use AmazonRegisterDataSource()" );

	amazonKey.setDataSource( datasource );

	AmazonS3 s3Client = new AmazonBase().getAmazonS3( amazonKey );

	GetObjectRequest gor = new GetObjectRequest( bucket, key );
	if ( props.containsKey( "aes256key" ) ) {
		String aes256key = props.getData( "aes256key" ).getString();

		if ( !aes256key.isEmpty() )
			gor.setSSECustomerKey( new SSECustomerKey( aes256key ) );
	}

	// Get the object
	try {
	
		S3Object s3object = s3Client.getObject( gor );

		_Session.setContentType( s3object.getObjectMetadata().getContentType() );

		InputStream in = s3object.getObjectContent();

		byte[] buffer = new byte[65536];
		int readCount = 0;

		while ( ( readCount = in.read( buffer ) ) != -1 ) {
			_Session.write( buffer, 0, readCount );
			_Session.pageFlush();
		}

	} catch ( Exception e ) {
		
		if ( e.getMessage().indexOf("404") != -1 ){
			_Session.setStatus( 404 );
			return;
		}else{
			cfEngine.log( e.getMessage() );
			throw newRunTimeException( e.getMessage() + "; key=" + key + "; bucket=" + bucket );
		}
	}
}
 

public String handleRequest(S3Event s3event, Context context) {
    _logger = context.getLogger();
    _logger.log("Received S3 Event: " + s3event.toJson());

    try {
        S3EventNotificationRecord record = s3event.getRecords().get(0);

        String bucket = record.getS3().getBucket().getName();
        String extractBucket = "extracts." + bucket;

        // Object key may have spaces or unicode non-ASCII characters.
        String key = URLDecoder.decode(record.getS3().getObject().getKey().replace('+', ' '), "UTF-8");

        // Short-circuit ignore .extract files because they have already been extracted, this prevents an endless loop
        if (key.toLowerCase().endsWith(".extract")) {
          _logger.log("Ignoring extract file " + key);
          return "Ignored";
        }

        AmazonS3 s3Client = new AmazonS3Client();
        S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucket, key));

        try (InputStream objectData = s3Object.getObjectContent()) {
            String extractJson = doTikaStuff(bucket, key, objectData);

            byte[] extractBytes = extractJson.getBytes(Charset.forName("UTF-8"));
            int extractLength = extractBytes.length;

            ObjectMetadata metaData = new ObjectMetadata();
            metaData.setContentLength(extractLength);

            _logger.log("Saving extract file to S3");
            InputStream inputStream = new ByteArrayInputStream(extractBytes);
            s3Client.putObject(extractBucket, key + ".extract", inputStream, metaData);
        }
    } catch (IOException | TransformerConfigurationException | SAXException e) {
        _logger.log("Exception: " + e.getLocalizedMessage());
        throw new RuntimeException(e);
    }
    return "Success";
}
 

public static void main(String[] args) throws IOException {
    //set-up the client
    AmazonS3 s3client = AmazonS3ClientBuilder
      .standard()
      .withCredentials(new AWSStaticCredentialsProvider(credentials))
      .withRegion(Regions.US_EAST_2)
      .build();
    
    AWSS3Service awsService = new AWSS3Service(s3client);

    bucketName = "baeldung-bucket";

    //creating a bucket
    if(awsService.doesBucketExist(bucketName)) {
        System.out.println("Bucket name is not available."
          + " Try again with a different Bucket name.");
        return;
    }
    awsService.createBucket(bucketName);
    
    //list all the buckets
    for(Bucket s : awsService.listBuckets() ) {
        System.out.println(s.getName());
    }
    
    //deleting bucket
    awsService.deleteBucket("baeldung-bucket-test2");
    
    //uploading object
    awsService.putObject(
      bucketName, 
      "Document/hello.txt",
      new File("/Users/user/Document/hello.txt")
    );

    //listing objects
    ObjectListing objectListing = awsService.listObjects(bucketName);
    for(S3ObjectSummary os : objectListing.getObjectSummaries()) {
        System.out.println(os.getKey());
    }

    //downloading an object
    S3Object s3object = awsService.getObject(bucketName, "Document/hello.txt");
    S3ObjectInputStream inputStream = s3object.getObjectContent();
    FileUtils.copyInputStreamToFile(inputStream, new File("/Users/user/Desktop/hello.txt"));
    
    //copying an object
    awsService.copyObject(
      "baeldung-bucket", 
      "picture/pic.png", 
      "baeldung-bucket2", 
      "Document/picture.png"
    );
    
    //deleting an object
    awsService.deleteObject(bucketName, "Document/hello.txt");

    //deleting multiple objects
    String objkeyArr[] = {
      "Document/hello2.txt", 
      "Document/picture.png"
    };
    
    DeleteObjectsRequest delObjReq = new DeleteObjectsRequest("baeldung-bucket")
      .withKeys(objkeyArr);
    awsService.deleteObjects(delObjReq);
}
 

@Test
public void testCopyObjectReplaceMetadata() throws Exception {
    String fromName = "from-name";
    String toName = "to-name";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        metadata.setCacheControl("max-age=3600");
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        metadata.setContentDisposition("attachment; filename=old.jpg");
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        metadata.setContentEncoding("compress");
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        metadata.setContentLanguage("en");
    }
    metadata.setContentType("audio/ogg");
    // TODO: expires
    metadata.setUserMetadata(ImmutableMap.of(
                    "key1", "value1",
                    "key2", "value2"));
    client.putObject(containerName, fromName, BYTE_SOURCE.openStream(),
            metadata);

    String cacheControl = "max-age=1800";
    String contentDisposition = "attachment; filename=new.jpg";
    String contentEncoding = "gzip";
    String contentLanguage = "fr";
    String contentType = "audio/mp4";
    ObjectMetadata contentMetadata = new ObjectMetadata();
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        contentMetadata.setCacheControl(cacheControl);
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        contentMetadata.setContentDisposition(contentDisposition);
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        contentMetadata.setContentEncoding(contentEncoding);
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        contentMetadata.setContentLanguage(contentLanguage);
    }
    contentMetadata.setContentType(contentType);
    // TODO: expires
    Map<String, String> userMetadata = ImmutableMap.of(
            "key3", "value3",
            "key4", "value4");
    contentMetadata.setUserMetadata(userMetadata);
    client.copyObject(new CopyObjectRequest(
                containerName, fromName, containerName, toName)
                        .withNewObjectMetadata(contentMetadata));

    S3Object object = client.getObject(containerName, toName);

    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }

    ObjectMetadata toContentMetadata = object.getObjectMetadata();
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        assertThat(contentMetadata.getCacheControl()).isEqualTo(
                cacheControl);
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        assertThat(toContentMetadata.getContentDisposition()).isEqualTo(
                contentDisposition);
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        assertThat(toContentMetadata.getContentEncoding()).isEqualTo(
                contentEncoding);
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        assertThat(toContentMetadata.getContentLanguage()).isEqualTo(
                contentLanguage);
    }
    assertThat(toContentMetadata.getContentType()).isEqualTo(
            contentType);
    // TODO: expires
    assertThat(toContentMetadata.getUserMetadata()).isEqualTo(
            userMetadata);
}
 

public static InputStream makeS3FileStream(String bucketName, String key) {
    AmazonS3 s3Client = makeS3Client();
    S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key));
    return object.getObjectContent();
}
 

public InputStream get(String bucket, String key) {
    GetObjectRequest request = new GetObjectRequest(bucket, key);
    S3Object response = client.getObject(request);
    return response.getObjectContent();
}
 

@Test
public void testMultipartCopy() throws Exception {
    // B2 requires two parts to issue an MPU
    assumeTrue(!blobStoreType.equals("b2"));

    String sourceBlobName = "testMultipartCopy-source";
    String targetBlobName = "testMultipartCopy-target";

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, sourceBlobName,
            BYTE_SOURCE.openStream(), metadata);

    InitiateMultipartUploadRequest initiateRequest =
            new InitiateMultipartUploadRequest(containerName,
                    targetBlobName);
    InitiateMultipartUploadResult initResult =
            client.initiateMultipartUpload(initiateRequest);
    String uploadId = initResult.getUploadId();

    CopyPartRequest copyRequest = new CopyPartRequest()
            .withDestinationBucketName(containerName)
            .withDestinationKey(targetBlobName)
            .withSourceBucketName(containerName)
            .withSourceKey(sourceBlobName)
            .withUploadId(uploadId)
            .withFirstByte(0L)
            .withLastByte(BYTE_SOURCE.size() - 1)
            .withPartNumber(1);
    CopyPartResult copyPartResult = client.copyPart(copyRequest);

    CompleteMultipartUploadRequest completeRequest =
            new CompleteMultipartUploadRequest(
                    containerName, targetBlobName, uploadId,
                    ImmutableList.of(copyPartResult.getPartETag()));
    client.completeMultipartUpload(completeRequest);

    S3Object object = client.getObject(containerName, targetBlobName);
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}