Java Code Examples for com.amazonaws.services.s3.model.PutObjectRequest#setMetadata()

private boolean uploadFile(File src, S3Details dstDetails, boolean archive) {
  PutObjectRequest request = new PutObjectRequest(dstDetails.getBucket(),
      FileUtil.appendPath(dstDetails.getKey(), src.getName()), src);
  ObjectMetadata meta = new ObjectMetadata();

  if (archive) {
    meta
        .setHeader(Headers.STORAGE_CLASS,
            StorageClass.fromValue(StringUtils.upperCase(archiveStorageClass)));
  }

  if (encrypt) {
    meta.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
  }

  request.setMetadata(meta);
  try {
    amazonS3.putObject(request);
    return true;
  } catch (Exception e) {
    log.error("Error while uploading file: {}", src, e);
  }

  return false;
}
 

@Override
public void transferProduct(Product product) throws DataTransferException, IOException {
	for (Reference ref : product.getProductReferences()) {
     String origRef = stripProtocol(ref.getOrigReference(), false);
	  String dataStoreRef = stripProtocol(ref.getDataStoreReference(), true);
		try {
		  PutObjectRequest request = new PutObjectRequest(
		      bucketName, dataStoreRef, new File(origRef));
		  if (encrypt) {
 				ObjectMetadata requestMetadata = new ObjectMetadata();
 				requestMetadata.setSSEAlgorithm(AES_256_SERVER_SIDE_ENCRYPTION);
 				request.setMetadata(requestMetadata);
		  }
       s3Client.putObject(request);
		} catch (AmazonClientException e) {
			throw new DataTransferException(String.format(
			    "Failed to upload product reference %s to S3 at %s", origRef,
			    dataStoreRef), e);
		}
	}
}
 

@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
  String json = note.toJson();
  String key = user + "/" + "notebook" + "/" + note.getId() + "/" + "note.json";

  File file = File.createTempFile("note", "json");
  try {
    Writer writer = new OutputStreamWriter(new FileOutputStream(file));
    writer.write(json);
    writer.close();

    PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);

    if (useServerSideEncryption) {
      // Request server-side encryption.
      ObjectMetadata objectMetadata = new ObjectMetadata();
      objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
      putRequest.setMetadata(objectMetadata);
    }
    if (objectCannedAcl != null) {
      putRequest.withCannedAcl(objectCannedAcl);
    }
    s3client.putObject(putRequest);
  }
  catch (AmazonClientException ace) {
    throw new IOException("Unable to store note in S3: " + ace, ace);
  }
  finally {
    FileUtils.deleteQuietly(file);
  }
}
 

@Override
public void save(Note note, AuthenticationInfo subject) throws IOException {
  String json = note.toJson();
  String key = rootFolder + "/" + buildNoteFileName(note);
  File file = File.createTempFile("note", "zpln");
  try {
    Writer writer = new OutputStreamWriter(new FileOutputStream(file));
    writer.write(json);
    writer.close();
    PutObjectRequest putRequest = new PutObjectRequest(bucketName, key, file);
    if (useServerSideEncryption) {
      // Request server-side encryption.
      ObjectMetadata objectMetadata = new ObjectMetadata();
      objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
      putRequest.setMetadata(objectMetadata);
    }
    if (objectCannedAcl != null) {
        putRequest.withCannedAcl(objectCannedAcl);
    }
    s3client.putObject(putRequest);
  }
  catch (AmazonClientException ace) {
    throw new IOException("Fail to store note: " + note.getPath() + " in S3", ace);
  }
  finally {
    FileUtils.deleteQuietly(file);
  }
}
 

@Override
public synchronized void close() throws IOException {
  if (closed) {
    return;
  }

  backupStream.close();
  if (LOG.isDebugEnabled()) {
    LOG.debug("OutputStream for key '" + key + "' closed. Now beginning upload");
    LOG.debug("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold);
  }


  try {
    final ObjectMetadata om = new ObjectMetadata();
    if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
      om.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
    putObjectRequest.setCannedAcl(cannedACL);
    putObjectRequest.setMetadata(om);

    Upload upload = transfers.upload(putObjectRequest);

    ProgressableProgressListener listener = 
      new ProgressableProgressListener(upload, progress, statistics);
    upload.addProgressListener(listener);

    upload.waitForUploadResult();

    long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred();
    if (statistics != null && delta != 0) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("S3A write delta changed after finished: " + delta + " bytes");
      }
      statistics.incrementBytesWritten(delta);
    }

    // This will delete unnecessary fake parent directories
    fs.finishedWrite(key);
  } catch (InterruptedException e) {
    throw new IOException(e);
  } finally {
    if (!backupFile.delete()) {
      LOG.warn("Could not delete temporary s3a file: {}", backupFile);
    }
    super.close();
    closed = true;
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("OutputStream for key '" + key + "' upload complete");
  }
}
 

/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name.
 *
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 *
 * delSrc indicates if the source should be removed
 * @param delSrc whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src path
 * @param dst path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, 
  Path dst) throws IOException {
  String key = pathToKey(dst);

  if (!overwrite && exists(dst)) {
    throw new IOException(dst + " already exists");
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Copying local file from " + src + " to " + dst);
  }

  // Since we have a local file, we don't need to stream into a temporary file
  LocalFileSystem local = getLocal(getConf());
  File srcfile = local.pathToFile(src);

  final ObjectMetadata om = new ObjectMetadata();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    om.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
  putObjectRequest.setCannedAcl(cannedACL);
  putObjectRequest.setMetadata(om);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Upload up = transfers.upload(putObjectRequest);
  up.addProgressListener(progressListener);
  try {
    up.waitForUploadResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }

  // This will delete unnecessary fake parent directories
  finishedWrite(key);

  if (delSrc) {
    local.delete(src, false);
  }
}
 

@Override
public synchronized void close() throws IOException {
  if (closed) {
    return;
  }

  backupStream.close();
  if (LOG.isDebugEnabled()) {
    LOG.debug("OutputStream for key '" + key + "' closed. Now beginning upload");
    LOG.debug("Minimum upload part size: " + partSize + " threshold " + partSizeThreshold);
  }


  try {
    final ObjectMetadata om = new ObjectMetadata();
    if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
      om.setServerSideEncryption(serverSideEncryptionAlgorithm);
    }
    PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, backupFile);
    putObjectRequest.setCannedAcl(cannedACL);
    putObjectRequest.setMetadata(om);

    Upload upload = transfers.upload(putObjectRequest);

    ProgressableProgressListener listener = 
      new ProgressableProgressListener(upload, progress, statistics);
    upload.addProgressListener(listener);

    upload.waitForUploadResult();

    long delta = upload.getProgress().getBytesTransferred() - listener.getLastBytesTransferred();
    if (statistics != null && delta != 0) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("S3A write delta changed after finished: " + delta + " bytes");
      }
      statistics.incrementBytesWritten(delta);
    }

    // This will delete unnecessary fake parent directories
    fs.finishedWrite(key);
  } catch (InterruptedException e) {
    throw new IOException(e);
  } finally {
    if (!backupFile.delete()) {
      LOG.warn("Could not delete temporary s3a file: {}", backupFile);
    }
    super.close();
    closed = true;
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("OutputStream for key '" + key + "' upload complete");
  }
}
 

/**
 * The src file is on the local disk.  Add it to FS at
 * the given dst name.
 *
 * This version doesn't need to create a temporary file to calculate the md5.
 * Sadly this doesn't seem to be used by the shell cp :(
 *
 * delSrc indicates if the source should be removed
 * @param delSrc whether to delete the src
 * @param overwrite whether to overwrite an existing file
 * @param src path
 * @param dst path
 */
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src, 
  Path dst) throws IOException {
  String key = pathToKey(dst);

  if (!overwrite && exists(dst)) {
    throw new IOException(dst + " already exists");
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("Copying local file from " + src + " to " + dst);
  }

  // Since we have a local file, we don't need to stream into a temporary file
  LocalFileSystem local = getLocal(getConf());
  File srcfile = local.pathToFile(src);

  final ObjectMetadata om = new ObjectMetadata();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    om.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key, srcfile);
  putObjectRequest.setCannedAcl(cannedACL);
  putObjectRequest.setMetadata(om);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Upload up = transfers.upload(putObjectRequest);
  up.addProgressListener(progressListener);
  try {
    up.waitForUploadResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }

  // This will delete unnecessary fake parent directories
  finishedWrite(key);

  if (delSrc) {
    local.delete(src, false);
  }
}
 

/**
 * Upload the current block as a single PUT request; if the buffer is empty a
 * 0-byte PUT will be invoked, as it is needed to create an entry at the far
 * end.
 *
 * @throws IOException any problem
 */
private void putObject() throws IOException {
  LOG.debug("Executing regular upload for {}", writeOperationHelper);

  final COSDataBlocks.DataBlock block = getActiveBlock();
  int size = block.dataSize();
  final COSDataBlocks.BlockUploadData uploadData = block.startUpload();
  final PutObjectRequest putObjectRequest = uploadData.hasFile()
      ? writeOperationHelper.newPutRequest(uploadData.getFile())
      : writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size);

  final ObjectMetadata om = new ObjectMetadata();
  om.setUserMetadata(mMetadata);
  if (contentType != null && !contentType.isEmpty()) {
    om.setContentType(contentType);
  } else {
    om.setContentType("application/octet-stream");
  }
  // if atomic write is enabled use the etag to ensure put request is atomic
  if (mAtomicWriteEnabled) {
    if (mEtag != null) {
      LOG.debug("Atomic write - setting If-Match header");
      om.setHeader("If-Match", mEtag);
    } else {
      LOG.debug("Atomic write - setting If-None-Match header");
      om.setHeader("If-None-Match", "*");
    }
  }
  putObjectRequest.setMetadata(om);
  ListenableFuture<PutObjectResult> putObjectResult =
      executorService.submit(new Callable<PutObjectResult>() {
        @Override
        public PutObjectResult call() throws Exception {
          PutObjectResult result;
          try {
            // the putObject call automatically closes the input
            // stream afterwards.
            result = writeOperationHelper.putObject(putObjectRequest);
          } finally {
            closeAll(LOG, uploadData, block);
          }
          return result;
        }
      });
  clearActiveBlock();
  // wait for completion
  try {
    putObjectResult.get();
  } catch (InterruptedException ie) {
    LOG.warn("Interrupted object upload", ie);
    Thread.currentThread().interrupt();
  } catch (ExecutionException ee) {
    throw extractException("regular upload", key, ee);
  }
}
 

@Override
public MultipleFileUpload uploadFileList(String bucketName, String virtualDirectoryKeyPrefix, File directory, List<File> files,
    ObjectMetadataProvider metadataProvider, TransferManager transferManager)
{
    LOGGER.debug(
        "uploadFileList(): bucketName = " + bucketName + ", virtualDirectoryKeyPrefix = " + virtualDirectoryKeyPrefix + ", directory = " + directory +
            ", files = " + files);

    String directoryPath = directory.getAbsolutePath();

    long totalFileLength = 0;
    List<Upload> subTransfers = new ArrayList<>();
    for (File file : files)
    {
        // Get path to file relative to the specified directory
        String relativeFilePath = file.getAbsolutePath().substring(directoryPath.length());

        // Replace any backslashes (i.e. Windows separator) with a forward slash.
        relativeFilePath = relativeFilePath.replace("\\", "/");

        // Remove any leading slashes
        relativeFilePath = relativeFilePath.replaceAll("^/+", "");

        long fileLength = file.length();

        // Remove any trailing slashes
        virtualDirectoryKeyPrefix = virtualDirectoryKeyPrefix.replaceAll("/+$", "");

        String s3ObjectKey = virtualDirectoryKeyPrefix + "/" + relativeFilePath;
        totalFileLength += fileLength;

        PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, s3ObjectKey, file);

        ObjectMetadata objectMetadata = new ObjectMetadata();
        metadataProvider.provideObjectMetadata(null, objectMetadata);
        putObjectRequest.setMetadata(objectMetadata);

        putObject(putObjectRequest, transferManager.getAmazonS3Client());

        subTransfers.add(new UploadImpl(null, null, null, null));
    }

    TransferProgress progress = new TransferProgress();
    progress.setTotalBytesToTransfer(totalFileLength);
    progress.updateProgress(totalFileLength);

    MultipleFileUploadImpl multipleFileUpload = new MultipleFileUploadImpl(null, progress, null, virtualDirectoryKeyPrefix, bucketName, subTransfers);
    multipleFileUpload.setState(TransferState.Completed);
    return multipleFileUpload;
}
 

private void writeFile( AmazonKey amazonKey, String bucket, String key, Map<String, String> metadata, StorageClass storage, String localpath, int retry, int retryseconds, boolean deletefile, boolean background, String callback, String callbackdata, String appname, String acl, String aes256key, Map<String, String> customheaders ) throws Exception {
	File localFile = new File( localpath );
	if ( !localFile.isFile() )
		throw new Exception( "The file specified does not exist: " + localpath );

	// Push this to the background loader to handle and return immediately
	if ( background ) {
		BackgroundUploader.acceptFile( amazonKey, bucket, key, metadata, storage, localpath, retry, retryseconds, deletefile, callback, callbackdata, appname, acl, aes256key, customheaders );
		return;
	}


	// Setup the object data
	ObjectMetadata omd = new ObjectMetadata();
	if ( metadata != null )
		omd.setUserMetadata( metadata );

	AmazonS3 s3Client = getAmazonS3( amazonKey );

	// Let us run around the number of attempts
	int attempts = 0;
	while ( attempts < retry ) {
		try {

			PutObjectRequest por = new PutObjectRequest( bucket, key, localFile );
			por.setMetadata( omd );
			por.setStorageClass( storage );

			if ( acl != null && !acl.isEmpty() )
				por.setCannedAcl( amazonKey.getAmazonCannedAcl( acl ) );

			if ( aes256key != null && !aes256key.isEmpty() )
				por.setSSECustomerKey( new SSECustomerKey( aes256key ) );

			if ( customheaders != null && !customheaders.isEmpty() ) {
				Iterator<String> it = customheaders.keySet().iterator();
				while ( it.hasNext() ) {
					String k = it.next();
					por.putCustomRequestHeader( k, customheaders.get( k ) );
				}
			}

			s3Client.putObject( por );
			break;

		} catch ( Exception e ) {
			cfEngine.log( "Failed: AmazonS3Write(bucket=" + bucket + "key=" + key + "; file=" + localFile + "; attempt=" + ( attempts + 1 ) + "; exception=" + e.getMessage() + ")" );
			attempts++;

			if ( attempts == retry )
				throw e;
			else
				Thread.sleep( retryseconds * 1000 );
		}
	}


	// delete the file now that it is a success
	if ( deletefile )
		localFile.delete();
}
 

private void enableS3Encryption(PutObjectRequest uploadRequest) {
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    uploadRequest.setMetadata(objectMetadata);
}
 

/**
 * Put any file to Amazon S3 storage.
 * 
 * @param bucket
 *            - S3 bucket name
 * @param key
 *            - S3 storage path. Example:
 *            DEMO/TestSuiteName/TestMethodName/file.txt
 * @param filePath
 *            - local storage path. Example: C:/Temp/file.txt
 * @param metadata
 *            - custom tags metadata like name etc
 * 
 */
public void put(String bucket, String key, String filePath, ObjectMetadata metadata) {

    /*
     * if (mode != S3Mode.WRITE) {
     * if (mode == S3Mode.READ) {
     * LOGGER.warn("Unable to put data in READ mode!");
     * }
     * return;
     * }
     */

    if (key == null) {
        throw new RuntimeException("Key is null!");
    }
    if (key.isEmpty()) {
        throw new RuntimeException("Key is empty!");
    }

    if (filePath == null) {
        throw new RuntimeException("FilePath is null!");
    }
    if (filePath.isEmpty()) {
        throw new RuntimeException("FilePath is empty!");
    }

    File file = new File(filePath);
    if (!file.exists()) {
        throw new RuntimeException("File does not exist! " + filePath);
    }

    try {
        LOGGER.debug("Uploading a new object to S3 from a file: "
                + filePath);

        PutObjectRequest object = new PutObjectRequest(bucket, key, file);
        if (metadata != null) {
            object.setMetadata(metadata);
        }

        s3client.putObject(object);
        LOGGER.debug("Uploaded to S3: '" + filePath + "' with key '" + key
                + "'");

    } catch (AmazonServiceException ase) {
        LOGGER.error("Caught an AmazonServiceException, which "
                + "means your request made it "
                + "to Amazon S3, but was rejected with an error response for some reason.\n"
                + "Error Message:    " + ase.getMessage() + "\n"
                + "HTTP Status Code: " + ase.getStatusCode() + "\n"
                + "AWS Error Code:   " + ase.getErrorCode() + "\n"
                + "Error Type:       " + ase.getErrorType() + "\n"
                + "Request ID:       " + ase.getRequestId());
    } catch (AmazonClientException ace) {
        LOGGER.error("Caught an AmazonClientException, which "
                + "means the client encountered "
                + "an internal error while trying to "
                + "communicate with S3, "
                + "such as not being able to access the network.\n"
                + "Error Message: " + ace.getMessage());
    }
}
 

/**
 * Create a putObject request.
 * Adds the ACL and metadata
 * @param key key of object
 * @param metadata metadata header
 * @param srcfile source file
 * @return the request
 */
public PutObjectRequest newPutObjectRequest(String key,
    ObjectMetadata metadata, File srcfile) {
  PutObjectRequest putObjectRequest = new PutObjectRequest(mBucket, key,
      srcfile);
  putObjectRequest.setMetadata(metadata);
  return putObjectRequest;
}