com.amazonaws.services.s3.model.CopyObjectRequest Java Examples

The following examples show how to use com.amazonaws.services.s3.model.CopyObjectRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3FileObjectTest.java    From hop with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoRename() throws Exception {
  String someNewBucketName = "someNewBucketName";
  String someNewKey = "some/newKey";
  S3FileName newFileName = new S3FileName( SCHEME, someNewBucketName, someNewBucketName + "/" + someNewKey, FileType.FILE );
  S3FileObject newFile = new S3FileObject( newFileName, fileSystemSpy );
  ArgumentCaptor<CopyObjectRequest> copyObjectRequestArgumentCaptor = ArgumentCaptor.forClass( CopyObjectRequest.class );
  when( s3ServiceMock.doesBucketExistV2( someNewBucketName ) ).thenReturn( true );
  s3FileObjectFileSpy.doAttach();
  s3FileObjectFileSpy.moveTo( newFile );

  verify( s3ServiceMock ).copyObject( copyObjectRequestArgumentCaptor.capture() );
  assertEquals( someNewBucketName, copyObjectRequestArgumentCaptor.getValue().getDestinationBucketName() );
  assertEquals( someNewKey, copyObjectRequestArgumentCaptor.getValue().getDestinationKey() );
  assertEquals( BUCKET_NAME, copyObjectRequestArgumentCaptor.getValue().getSourceBucketName() );
  assertEquals( origKey, copyObjectRequestArgumentCaptor.getValue().getSourceKey() );
}
 
Example #2
Source File: S3NFileObjectTest.java    From hop with Apache License 2.0 6 votes vote down vote up
@Test
public void testDoRename() throws Exception {
  String someNewBucketName = "someNewBucketName";
  String someNewKey = "some/newKey";
  S3NFileName newFileName = new S3NFileName( SCHEME, someNewBucketName, "/" + someNewBucketName + "/" + someNewKey, FileType.FILE );
  S3NFileObject newFile = new S3NFileObject( newFileName, fileSystemSpy );
  ArgumentCaptor<CopyObjectRequest> copyObjectRequestArgumentCaptor = ArgumentCaptor.forClass( CopyObjectRequest.class );
  when( s3ServiceMock.doesBucketExistV2( someNewBucketName ) ).thenReturn( true );
  s3FileObjectFileSpy.moveTo( newFile );

  verify( s3ServiceMock ).copyObject( copyObjectRequestArgumentCaptor.capture() );
  assertEquals( someNewBucketName, copyObjectRequestArgumentCaptor.getValue().getDestinationBucketName() );
  assertEquals( someNewKey, copyObjectRequestArgumentCaptor.getValue().getDestinationKey() );
  assertEquals( BUCKET_NAME, copyObjectRequestArgumentCaptor.getValue().getSourceBucketName() );
  assertEquals( origKey, copyObjectRequestArgumentCaptor.getValue().getSourceKey() );
}
 
Example #3
Source File: S3Backuper.java    From cassandra-backup with Apache License 2.0 6 votes vote down vote up
@Override
public FreshenResult freshenRemoteObject(final RemoteObjectReference object) throws InterruptedException {
    final String canonicalPath = ((S3RemoteObjectReference) object).canonicalPath;

    final CopyObjectRequest copyRequest = new CopyObjectRequest(request.storageLocation.bucket,
                                                                canonicalPath,
                                                                request.storageLocation.bucket,
                                                                canonicalPath).withStorageClass(StorageClass.Standard);

    try {
        // attempt to refresh existing object in the bucket via an inplace copy
        transferManager.copy(copyRequest).waitForCompletion();
        return FreshenResult.FRESHENED;

    } catch (final AmazonServiceException e) {
        // AWS S3 under certain access policies can't return NoSuchKey (404)
        // instead, it returns AccessDenied (403) — handle it the same way
        if (e.getStatusCode() != 404 && e.getStatusCode() != 403) {
            throw e;
        }

        // the freshen failed because the file/key didn't exist
        return FreshenResult.UPLOAD_REQUIRED;
    }
}
 
Example #4
Source File: AmazonS3FileSystem.java    From iaf with Apache License 2.0 6 votes vote down vote up
/**
 * Copies a file from one Amazon S3 bucket to another one. 
 *
 * @param fileName
 * 				This is the name of the file that is desired to be copied.
 * 
 * @param destinationFileName
 * 				The name of the destination file
 */
public String copyObject(String fileName, String destinationFileName) throws SenderException {
	try {
		bucketDoesNotExist(bucketName); //if bucket does not exists this method throws an exception
		fileDoesNotExist(bucketName, fileName); //if object does not exists this method throws an exception
		if (!s3Client.doesBucketExistV2(destinationBucketName))
			bucketCreationWithObjectAction(destinationBucketName);
		if (!s3Client.doesObjectExist(destinationBucketName, destinationFileName)) {
			CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, fileName, destinationBucketName, destinationFileName);
			if (isStorageClassEnabled())
				copyObjectRequest.setStorageClass(getStorageClass());
			s3Client.copyObject(copyObjectRequest);
			log.debug("Object with fileName [" + fileName + "] copied from bucket with bucketName [" + bucketName
					+ "] into bucket with bucketName [" + destinationBucketName + "] and new fileName ["
					+ destinationFileName + "]");
		} else
			throw new SenderException(" file with given name already exists, please specify a new name");
	} catch (AmazonServiceException e) {
		log.error("Failed to perform [copy] action on object with fileName [" + fileName + "]");
		throw new SenderException("Failed to perform [copy] action on object with fileName [" + fileName + "]");
	}

	return destinationFileName;
}
 
Example #5
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void copyDefaultCopierOptions() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);

  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  ArgumentCaptor<CopyObjectRequest> argument = ArgumentCaptor.forClass(CopyObjectRequest.class);
  verify(mockedTransferManager).copy(argument.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
  CopyObjectRequest copyObjectRequest = argument.getValue();
  assertNull(copyObjectRequest.getNewObjectMetadata());
}
 
Example #6
Source File: S3FileSystem.java    From beam with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
CopyObjectResult atomicCopy(
    S3ResourceId sourcePath, S3ResourceId destinationPath, ObjectMetadata sourceObjectMetadata)
    throws AmazonClientException {
  CopyObjectRequest copyObjectRequest =
      new CopyObjectRequest(
          sourcePath.getBucket(),
          sourcePath.getKey(),
          destinationPath.getBucket(),
          destinationPath.getKey());
  copyObjectRequest.setNewObjectMetadata(sourceObjectMetadata);
  copyObjectRequest.setStorageClass(options.getS3StorageClass());
  copyObjectRequest.setSourceSSECustomerKey(options.getSSECustomerKey());
  copyObjectRequest.setDestinationSSECustomerKey(options.getSSECustomerKey());
  return amazonS3.get().copyObject(copyObjectRequest);
}
 
Example #7
Source File: S3S3Copier.java    From circus-train with Apache License 2.0 6 votes vote down vote up
private Copy submitCopyJob(CopyJobRequest copyJob) {
  CopyObjectRequest copyObjectRequest = copyJob.getCopyObjectRequest();
  LOG
      .info("Copying object from '{}/{}' to '{}/{}'", copyObjectRequest.getSourceBucketName(),
          copyObjectRequest.getSourceKey(), copyObjectRequest.getDestinationBucketName(),
          copyObjectRequest.getDestinationKey());
  return transferManager.copy(copyObjectRequest, srcClient, copyJob.getTransferStateChangeListener());
}
 
Example #8
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}
 
Example #9
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 6 votes vote down vote up
@Test
public void copyCheckTransferManagerIsShutdownWhenSubmittingJobExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenThrow(new AmazonServiceException("MyCause"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), startsWith("MyCause"));
  }
}
 
Example #10
Source File: S3CommonFileObject.java    From hop with Apache License 2.0 6 votes vote down vote up
@Override
protected void doRename( FileObject newFile ) throws Exception {
  // no folder renames on S3
  if ( getType().equals( FileType.FOLDER ) ) {
    throw new FileSystemException( "vfs.provider/rename-not-supported.error" );
  }

  s3ObjectMetadata = fileSystem.getS3Client().getObjectMetadata( bucketName, key );

  if ( s3ObjectMetadata == null ) {
    // object doesn't exist
    throw new FileSystemException( "vfs.provider/rename.error", this, newFile );
  }

  S3CommonFileObject dest = (S3CommonFileObject) newFile;

  // 1. copy the file
  CopyObjectRequest copyObjRequest = createCopyObjectRequest( bucketName, key, dest.bucketName, dest.key );
  fileSystem.getS3Client().copyObject( copyObjRequest );

  // 2. delete self
  delete();
}
 
Example #11
Source File: S3ArchiveImageHandler.java    From smart-security-camera with GNU General Public License v3.0 5 votes vote down vote up
@Override
public Parameters handleRequest(Parameters parameters, Context context) {

    context.getLogger().log("Input Function [" + context.getFunctionName() + "], Parameters [" + parameters + "]");

    // The archive location of the snapshot will be decided by the alert
    // flag
    String newFilename;
    if (parameters.getSendAlert()) {
        newFilename =  parameters.getS3Key().replace("upload/", "archive/alerts/");
    } else {
        newFilename = parameters.getS3Key().replace("upload/", "archive/falsepositives/");
    }

    // Ensure that the first two hyphens are used to create sub-directories
    // in the file path
    newFilename = newFilename.replaceFirst("-", "/");
    newFilename = newFilename.replaceFirst("-", "/");

    // Using the S3 client, first copy the file to the archive, and then
    // delete the original
    AmazonS3 client = AmazonS3ClientBuilder.defaultClient();
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(parameters.getS3Bucket(), parameters.getS3Key(), parameters.getS3Bucket(), newFilename);
    client.copyObject(copyObjectRequest);
    DeleteObjectRequest deleteObjectRequest = new DeleteObjectRequest(parameters.getS3Bucket(), parameters.getS3Key());
    client.deleteObject(deleteObjectRequest);

    // Place the new location in the parameters
    parameters.setS3ArchivedKey(newFilename);

    context.getLogger().log("Output Function [" + context.getFunctionName() + "], Parameters [" + parameters + "]");

    return parameters;
}
 
Example #12
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void copyCannedAcl() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  Map<String, Object> copierOptions = new HashMap<>();
  copierOptions
      .put(S3S3CopierOptions.Keys.CANNED_ACL.keyName(), CannedAccessControlList.BucketOwnerFullControl.toString());
  S3S3CopierOptions customOptions = new S3S3CopierOptions(copierOptions);

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(customOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);

  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, customOptions);
  s3s3Copier.copy();
  ArgumentCaptor<CopyObjectRequest> argument = ArgumentCaptor.forClass(CopyObjectRequest.class);
  verify(mockedTransferManager).copy(argument.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
  CopyObjectRequest copyObjectRequest = argument.getValue();
  assertThat(copyObjectRequest.getCannedAccessControlList(), is(CannedAccessControlList.BucketOwnerFullControl));
}
 
Example #13
Source File: S3AFileSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void copyFile(String srcKey, String dstKey) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("copyFile " + srcKey + " -> " + dstKey);
  }

  ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
  final ObjectMetadata dstom = srcom.clone();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    dstom.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
  copyObjectRequest.setCannedAccessControlList(cannedACL);
  copyObjectRequest.setNewObjectMetadata(dstom);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Copy copy = transfers.copy(copyObjectRequest);
  copy.addProgressListener(progressListener);
  try {
    copy.waitForCopyResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }
}
 
Example #14
Source File: S3AFileSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void copyFile(String srcKey, String dstKey) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("copyFile " + srcKey + " -> " + dstKey);
  }

  ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
  final ObjectMetadata dstom = srcom.clone();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    dstom.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
  copyObjectRequest.setCannedAccessControlList(cannedACL);
  copyObjectRequest.setNewObjectMetadata(dstom);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Copy copy = transfers.copy(copyObjectRequest);
  copy.addProgressListener(progressListener);
  try {
    copy.waitForCopyResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }
}
 
Example #15
Source File: S3FileSystemTest.java    From beam with Apache License 2.0 5 votes vote down vote up
private void testAtomicCopy(S3Options options) {
  S3FileSystem s3FileSystem = buildMockedS3FileSystem(options);

  S3ResourceId sourcePath = S3ResourceId.fromUri("s3://bucket/from");
  S3ResourceId destinationPath = S3ResourceId.fromUri("s3://bucket/to");

  CopyObjectResult copyObjectResult = new CopyObjectResult();
  if (getSSECustomerKeyMd5(options) != null) {
    copyObjectResult.setSSECustomerKeyMd5(getSSECustomerKeyMd5(options));
  }
  CopyObjectRequest copyObjectRequest =
      new CopyObjectRequest(
          sourcePath.getBucket(),
          sourcePath.getKey(),
          destinationPath.getBucket(),
          destinationPath.getKey());
  copyObjectRequest.setSourceSSECustomerKey(options.getSSECustomerKey());
  copyObjectRequest.setDestinationSSECustomerKey(options.getSSECustomerKey());
  when(s3FileSystem.getAmazonS3Client().copyObject(any(CopyObjectRequest.class)))
      .thenReturn(copyObjectResult);
  assertEquals(
      getSSECustomerKeyMd5(options),
      s3FileSystem.getAmazonS3Client().copyObject(copyObjectRequest).getSSECustomerKeyMd5());

  ObjectMetadata sourceS3ObjectMetadata = new ObjectMetadata();
  s3FileSystem.atomicCopy(sourcePath, destinationPath, sourceS3ObjectMetadata);

  verify(s3FileSystem.getAmazonS3Client(), times(2)).copyObject(any(CopyObjectRequest.class));
}
 
Example #16
Source File: Rename.java    From openbd-core with GNU General Public License v3.0 5 votes vote down vote up
public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException{
AmazonKey amazonKey	= getAmazonKey(_session, argStruct);
AmazonS3 s3Client		= getAmazonS3(amazonKey);

	String bucket			= getNamedStringParam(argStruct, "bucket", null );
	String srckey			= getNamedStringParam(argStruct, "srckey", null );
	String deskey			= getNamedStringParam(argStruct, "destkey", null );
	String aes256key	= getNamedStringParam(argStruct, "aes256key", null );

if ( srckey != null && srckey.charAt( 0 ) == '/' )
	srckey	= srckey.substring(1);

if ( deskey != null && deskey.charAt( 0 ) == '/' )
	deskey	= deskey.substring(1);

	
	CopyObjectRequest cor = new CopyObjectRequest(bucket, srckey, bucket, deskey);
	
	if ( aes256key != null && !aes256key.isEmpty() ){
		cor.setSourceSSECustomerKey( new SSECustomerKey(aes256key) );
		cor.setDestinationSSECustomerKey( new SSECustomerKey(aes256key) );
	}
	

try {
	s3Client.copyObject(cor);
	s3Client.deleteObject(new DeleteObjectRequest(bucket, srckey));
	return cfBooleanData.TRUE;
} catch (Exception e) {
	throwException(_session, "AmazonS3: " + e.getMessage() );
	return cfBooleanData.FALSE;
}
}
 
Example #17
Source File: AmazonS3Util.java    From datacollector with Apache License 2.0 5 votes vote down vote up
static void copy(
    AmazonS3 s3Client,
    String srcBucket,
    String sourceKey,
    String destBucket,
    String destKey,
    boolean isMove
) {
  CopyObjectRequest cp = new CopyObjectRequest(srcBucket, sourceKey, destBucket, destKey);
  s3Client.copyObject(cp);
  if (isMove) {
    s3Client.deleteObject(new DeleteObjectRequest(srcBucket, sourceKey));
  }
}
 
Example #18
Source File: S3FileManagerImpl.java    From entrada with GNU General Public License v3.0 5 votes vote down vote up
@Override
public boolean move(String src, String dst, boolean archive) {
  log.info("Move: {} to {}", src, dst);
  // s3 has no move operation, so do a copy and delete
  // this is not an atomic operation
  Optional<S3Details> srcDetails = S3Details.from(src);
  Optional<S3Details> dstDetails = S3Details.from(dst);

  if (srcDetails.isPresent() && dstDetails.isPresent()) {
    CopyObjectRequest cor = new CopyObjectRequest(srcDetails.get().getBucket(),
        srcDetails.get().getKey(), dstDetails.get().getBucket(), dstDetails.get().getKey());
    // make sure to set the storage class for file copy
    if (archive) {
      // set class for archive file
      cor.setStorageClass(StorageClass.fromValue(StringUtils.upperCase(archiveStorageClass)));
    } else {
      // set class for parquet files
      cor.setStorageClass(StorageClass.fromValue(StringUtils.upperCase(uploadStorageClass)));
    }
    try {
      CopyObjectResult r = amazonS3.copyObject(cor);

      if (Objects.nonNull(r.getETag())) {
        // copy ok, delete src
        amazonS3.deleteObject(srcDetails.get().getBucket(), srcDetails.get().getKey());
      }

      return true;
    } catch (Exception e) {
      log.error("Error during copying {} to ", src, dst, e);
    }
  }
  return false;
}
 
Example #19
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void copySafelyShutDownTransferWhenRetryFails() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenThrow(new AmazonClientException("S3 error"));
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("Exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getMessage(), is("Error in S3S3Copier:"));
    assertThat(e.getCause().getMessage(), startsWith("S3 error"));
  }
}
 
Example #20
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void copyServerSideEncryption() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  Map<String, Object> copierOptions = new HashMap<>();
  copierOptions.put(S3S3CopierOptions.Keys.S3_SERVER_SIDE_ENCRYPTION.keyName(), "true");
  S3S3CopierOptions customOptions = new S3S3CopierOptions(copierOptions);

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(customOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);

  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, customOptions);
  s3s3Copier.copy();
  ArgumentCaptor<CopyObjectRequest> argument = ArgumentCaptor.forClass(CopyObjectRequest.class);
  verify(mockedTransferManager).copy(argument.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
  CopyObjectRequest copyObjectRequest = argument.getValue();
  assertThat(copyObjectRequest.getNewObjectMetadata().getSSEAlgorithm(),
      is(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION));
}
 
Example #21
Source File: S3S3CopierTest.java    From circus-train with Apache License 2.0 5 votes vote down vote up
@Test
public void copyCheckTransferManagerIsShutdownWhenMaxRetriesExceeded() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    verify(mockedTransferManager, Mockito.times(3))
        .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class));
    assertThat(e.getMessage(), is("1 job(s) failed the maximum number of copy attempts, 3"));
  }
}
 
Example #22
Source File: S3S3Copier.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private void applyObjectMetadata(CopyObjectRequest copyObjectRequest) {
  if (s3s3CopierOptions.isS3ServerSideEncryption()) {
    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION);
    copyObjectRequest.setNewObjectMetadata(objectMetadata);
  }
}
 
Example #23
Source File: S3S3Copier.java    From circus-train with Apache License 2.0 5 votes vote down vote up
private void initialiseCopyJobsFromListing(
    AmazonS3URI sourceS3Uri,
    final AmazonS3URI targetS3Uri,
    ListObjectsRequest request,
    ObjectListing listing) {
  LOG
      .debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
          request.getPrefix());
  List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
  for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
    totalBytesToReplicate += s3ObjectSummary.getSize();
    String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
    final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
        s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);

    if (s3s3CopierOptions.getCannedAcl() != null) {
      copyObjectRequest.withCannedAccessControlList(s3s3CopierOptions.getCannedAcl());
    }

    applyObjectMetadata(copyObjectRequest);

    TransferStateChangeListener stateChangeListener = new BytesTransferStateChangeListener(s3ObjectSummary,
        targetS3Uri, targetKey);
    copyJobRequests.add(new CopyJobRequest(copyObjectRequest, stateChangeListener));
  }
}
 
Example #24
Source File: S3KeyCopyingTOCPayloadHandler.java    From s3-bucket-loader with Apache License 2.0 5 votes vote down vote up
@Override
public void handlePayload(TOCPayload payload, WorkerState workerState) throws Exception {
	
	TocInfo tocInfo = payload.tocInfo;
	
	String logPrefix = "handlePayload() KeyCopy s3://" + this.sourceS3BucketName + "/" + tocInfo.path + 
			" => s3://" + this.targetS3BucketName +"/"+ tocInfo.path;
	
	try {
	
		CopyObjectRequest copyRequest = new CopyObjectRequest(this.sourceS3BucketName, 
															  tocInfo.path, 
															  this.targetS3BucketName, 
															  tocInfo.path);
		copyRequest.setStorageClass(storageClass);
		// copyRequest.setGeneralProgressListener(this);
		
		if (this.enableServerSideEncryption) {
			copyRequest.putCustomRequestHeader("x-amz-server-side-encryption", "AES256");
		}
		
		CopyObjectResult copyResult = s3Client.copyObject(copyRequest);
		
		logger.debug(logPrefix + " copied OK");
		workerState.addTocPathWritten(new TocPathOpResult(payload.mode, true, tocInfo.path, "s3.copyKey", "OK"));
		
	} catch(Exception e) {
		logger.error(logPrefix + " unexpected ERROR: " + e.getMessage(),e);
		workerState.addTocPathWriteFailure(
				new TocPathOpResult(payload.mode, false, tocInfo.path, "s3.copyKey", logPrefix + " " + e.getMessage()));
	}
	
}
 
Example #25
Source File: AwsSdkTest.java    From s3proxy with Apache License 2.0 4 votes vote down vote up
@Test
public void testCopyObjectReplaceMetadata() throws Exception {
    String fromName = "from-name";
    String toName = "to-name";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        metadata.setCacheControl("max-age=3600");
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        metadata.setContentDisposition("attachment; filename=old.jpg");
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        metadata.setContentEncoding("compress");
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        metadata.setContentLanguage("en");
    }
    metadata.setContentType("audio/ogg");
    // TODO: expires
    metadata.setUserMetadata(ImmutableMap.of(
                    "key1", "value1",
                    "key2", "value2"));
    client.putObject(containerName, fromName, BYTE_SOURCE.openStream(),
            metadata);

    String cacheControl = "max-age=1800";
    String contentDisposition = "attachment; filename=new.jpg";
    String contentEncoding = "gzip";
    String contentLanguage = "fr";
    String contentType = "audio/mp4";
    ObjectMetadata contentMetadata = new ObjectMetadata();
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        contentMetadata.setCacheControl(cacheControl);
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        contentMetadata.setContentDisposition(contentDisposition);
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        contentMetadata.setContentEncoding(contentEncoding);
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        contentMetadata.setContentLanguage(contentLanguage);
    }
    contentMetadata.setContentType(contentType);
    // TODO: expires
    Map<String, String> userMetadata = ImmutableMap.of(
            "key3", "value3",
            "key4", "value4");
    contentMetadata.setUserMetadata(userMetadata);
    client.copyObject(new CopyObjectRequest(
                containerName, fromName, containerName, toName)
                        .withNewObjectMetadata(contentMetadata));

    S3Object object = client.getObject(containerName, toName);

    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }

    ObjectMetadata toContentMetadata = object.getObjectMetadata();
    if (!Quirks.NO_CACHE_CONTROL_SUPPORT.contains(blobStoreType)) {
        assertThat(contentMetadata.getCacheControl()).isEqualTo(
                cacheControl);
    }
    if (!Quirks.NO_CONTENT_DISPOSITION.contains(blobStoreType)) {
        assertThat(toContentMetadata.getContentDisposition()).isEqualTo(
                contentDisposition);
    }
    if (!Quirks.NO_CONTENT_ENCODING.contains(blobStoreType)) {
        assertThat(toContentMetadata.getContentEncoding()).isEqualTo(
                contentEncoding);
    }
    if (!Quirks.NO_CONTENT_LANGUAGE.contains(blobStoreType)) {
        assertThat(toContentMetadata.getContentLanguage()).isEqualTo(
                contentLanguage);
    }
    assertThat(toContentMetadata.getContentType()).isEqualTo(
            contentType);
    // TODO: expires
    assertThat(toContentMetadata.getUserMetadata()).isEqualTo(
            userMetadata);
}
 
Example #26
Source File: S3CopyStep.java    From pipeline-aws-plugin with Apache License 2.0 4 votes vote down vote up
@Override
public String run() throws Exception {
	final String fromBucket = this.step.getFromBucket();
	final String toBucket = this.step.getToBucket();
	final String fromPath = this.step.getFromPath();
	final String toPath = this.step.getToPath();
	final String kmsId = this.step.getKmsId();
	final Map<String, String> metadatas = new HashMap<>();
	final CannedAccessControlList acl = this.step.getAcl();
	final String cacheControl = this.step.getCacheControl();
	final String contentType = this.step.getContentType();
	final String sseAlgorithm = this.step.getSseAlgorithm();
	final S3ClientOptions s3ClientOptions = this.step.createS3ClientOptions();
	final EnvVars envVars = this.getContext().get(EnvVars.class);

	if (this.step.getMetadatas() != null && this.step.getMetadatas().length != 0) {
		for (String metadata : this.step.getMetadatas()) {
			if (metadata.split(":").length == 2) {
				metadatas.put(metadata.split(":")[0], metadata.split(":")[1]);
			}
		}
	}

	Preconditions.checkArgument(fromBucket != null && !fromBucket.isEmpty(), "From bucket must not be null or empty");
	Preconditions.checkArgument(fromPath != null && !fromPath.isEmpty(), "From path must not be null or empty");
	Preconditions.checkArgument(toBucket != null && !toBucket.isEmpty(), "To bucket must not be null or empty");
	Preconditions.checkArgument(toPath != null && !toPath.isEmpty(), "To path must not be null or empty");

	TaskListener listener = Execution.this.getContext().get(TaskListener.class);
	listener.getLogger().format("Copying s3://%s/%s to s3://%s/%s%n", fromBucket, fromPath, toBucket, toPath);

	CopyObjectRequest request = new CopyObjectRequest(fromBucket, fromPath, toBucket, toPath);

	// Add metadata
	if (metadatas.size() > 0 || (cacheControl != null && !cacheControl.isEmpty()) || (contentType != null && !contentType.isEmpty()) || (sseAlgorithm != null && !sseAlgorithm.isEmpty())) {
		ObjectMetadata metas = new ObjectMetadata();
		if (metadatas.size() > 0) {
			metas.setUserMetadata(metadatas);
		}
		if (cacheControl != null && !cacheControl.isEmpty()) {
			metas.setCacheControl(cacheControl);
		}
		if (contentType != null && !contentType.isEmpty()) {
			metas.setContentType(contentType);
		}
		if (sseAlgorithm != null && !sseAlgorithm.isEmpty()) {
			metas.setSSEAlgorithm(sseAlgorithm);
		}
		request.withNewObjectMetadata(metas);
	}

	// Add acl
	if (acl != null) {
		request.withCannedAccessControlList(acl);
	}

	// Add kms
	if (kmsId != null && !kmsId.isEmpty()) {
		listener.getLogger().format("Using KMS: %s%n", kmsId);
		request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsId));
	}

	TransferManager mgr = TransferManagerBuilder.standard()
			.withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars))
			.build();
	try {
		final Copy copy = mgr.copy(request);
		copy.addProgressListener((ProgressListener) progressEvent -> {
			if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
				listener.getLogger().println("Finished: " + copy.getDescription());
			}
		});
		copy.waitForCompletion();
	}
	finally{
		mgr.shutdownNow();
	}

	listener.getLogger().println("Copy complete");
	return String.format("s3://%s/%s", toBucket, toPath);
}
 
Example #27
Source File: Copy.java    From openbd-core with GNU General Public License v3.0 4 votes vote down vote up
public cfData execute( cfSession _session, cfArgStructData argStruct ) throws cfmRunTimeException{

AmazonKey amazonKey	= getAmazonKey(_session, argStruct);
AmazonS3 s3Client		= getAmazonS3(amazonKey);

	String srcbucket		= getNamedStringParam(argStruct, "srcbucket", null );
	String srckey				= getNamedStringParam(argStruct, "srckey", null );
	String srcaes256key	= getNamedStringParam(argStruct, "srcaes256key", null );
	
	String destbucket					= getNamedStringParam(argStruct, "destbucket", null );
	String deskey							= getNamedStringParam(argStruct, "destkey", null );
	String destaes256key			= getNamedStringParam(argStruct, "destaes256key", null );
	String deststorageclass		= getNamedStringParam(argStruct, "deststorageclass", null );
	String destacl						= getNamedStringParam(argStruct, "destacl", null );
	
if ( srckey != null && srckey.charAt( 0 ) == '/' )
	srckey	= srckey.substring(1);

if ( deskey != null && deskey.charAt( 0 ) == '/' )
	deskey	= deskey.substring(1);
	
	CopyObjectRequest cor = new CopyObjectRequest(srcbucket, srckey, destbucket, deskey);
	
	if ( srcaes256key != null && !srcaes256key.isEmpty() )
		cor.setSourceSSECustomerKey( new SSECustomerKey(srcaes256key) );

	if ( destaes256key != null && !destaes256key.isEmpty() )
		cor.setDestinationSSECustomerKey( new SSECustomerKey(destaes256key) );
	
	if ( deststorageclass != null && !deststorageclass.isEmpty() )
		cor.setStorageClass( amazonKey.getAmazonStorageClass(deststorageclass) );
	
	if ( destacl != null && !destacl.isEmpty() )
		cor.setCannedAccessControlList( amazonKey.getAmazonCannedAcl(destacl) );
	
try {
	s3Client.copyObject(cor);
	return cfBooleanData.TRUE;
} catch (Exception e) {
	throwException(_session, "AmazonS3: " + e.getMessage() );
	return cfBooleanData.FALSE;
}
}
 
Example #28
Source File: S3Client.java    From Scribengin with GNU Affero General Public License v3.0 4 votes vote down vote up
public void updateObjectMetadata(String bucketName, String key, ObjectMetadata metadata) {
  CopyObjectRequest request = 
    new CopyObjectRequest(bucketName, key, bucketName, key).withNewObjectMetadata(metadata);
  s3Client.copyObject(request);
}
 
Example #29
Source File: DummyS3Client.java    From ignite with Apache License 2.0 4 votes vote down vote up
/** Unsupported Operation. */
@Override public CopyObjectResult copyObject(CopyObjectRequest cpObjReq) throws SdkClientException {
    throw new UnsupportedOperationException("Operation not supported");
}
 
Example #30
Source File: MockS3OperationsImpl.java    From herd with Apache License 2.0 4 votes vote down vote up
/**
 * {@inheritDoc} <p/> <p> This implementation simulates a copyFile operation. </p> <p> This method copies files in-memory. </p> <p> The result {@link Copy}
 * has the following properties: <dl> <p/> <dt>description</dt> <dd>"MockTransfer"</dd> <p/> <dt>state</dt> <dd>{@link TransferState#Completed}</dd> <p/>
 * <dt>transferProgress.totalBytesToTransfer</dt> <dd>1024</dd> <p/> <dt>transferProgress.updateProgress</dt> <dd>1024</dd> <p/> </dl> <p/> All other
 * properties are set as default. </p> <p> This operation takes the following hints when suffixed in copyObjectRequest.sourceKey: <dl> <p/>
 * <dt>MOCK_S3_FILE_NAME_SERVICE_EXCEPTION</dt> <dd>Throws a AmazonServiceException</dd> <p/> </dl> </p>
 */
@Override
public Copy copyFile(final CopyObjectRequest copyObjectRequest, TransferManager transferManager)
{
    LOGGER.debug(
        "copyFile(): copyObjectRequest.getSourceBucketName() = " + copyObjectRequest.getSourceBucketName() + ", copyObjectRequest.getSourceKey() = " +
            copyObjectRequest.getSourceKey() + ", copyObjectRequest.getDestinationBucketName() = " + copyObjectRequest.getDestinationBucketName() +
            ", copyObjectRequest.getDestinationKey() = " + copyObjectRequest.getDestinationKey());

    if (copyObjectRequest.getSourceKey().endsWith(MOCK_S3_FILE_NAME_SERVICE_EXCEPTION))
    {
        throw new AmazonServiceException(null);
    }

    String sourceBucketName = copyObjectRequest.getSourceBucketName();
    String sourceKey = copyObjectRequest.getSourceKey();

    MockS3Bucket mockSourceS3Bucket = getOrCreateBucket(sourceBucketName);
    MockS3Object mockSourceS3Object = mockSourceS3Bucket.getObjects().get(sourceKey);

    if (mockSourceS3Object == null)
    {
        AmazonServiceException amazonServiceException = new AmazonServiceException(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        amazonServiceException.setErrorCode(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        throw amazonServiceException;
    }

    // Set the result CopyImpl and TransferProgress.
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(mockSourceS3Object.getObjectMetadata().getContentLength());
    transferProgress.updateProgress(mockSourceS3Object.getObjectMetadata().getContentLength());
    CopyImpl copy = new CopyImpl(MOCK_TRANSFER_DESCRIPTION, transferProgress, null, null);
    copy.setState(TransferState.Completed);

    // If an invalid KMS Id was passed in, mark the transfer as failed and return an exception via the transfer monitor.
    if (copyObjectRequest.getSSEAwsKeyManagementParams() != null)
    {
        final String kmsId = copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId();
        if (kmsId.startsWith(MOCK_KMS_ID_FAILED_TRANSFER))
        {
            copy.setState(TransferState.Failed);
            copy.setMonitor(new TransferMonitor()
            {
                @Override
                public Future<?> getFuture()
                {
                    if (!kmsId.equals(MOCK_KMS_ID_FAILED_TRANSFER_NO_EXCEPTION))
                    {
                        throw new AmazonServiceException("Key '" + copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId() +
                            "' does not exist (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: 1234567890123456)");
                    }

                    // We don't want an exception to be thrown so return a basic future that won't throw an exception.
                    BasicFuture<?> future = new BasicFuture<Void>(null);
                    future.completed(null);
                    return future;
                }

                @Override
                public boolean isDone()
                {
                    return true;
                }
            });
        }
        else if (kmsId.startsWith(MOCK_KMS_ID_CANCELED_TRANSFER))
        {
            // If the KMS indicates a cancelled transfer, just update the state to canceled.
            copy.setState(TransferState.Canceled);
        }
    }

    // If copy operation is marked as completed, perform the actual file copy in memory.
    if (copy.getState().equals(TransferState.Completed))
    {
        String destinationBucketName = copyObjectRequest.getDestinationBucketName();
        String destinationObjectKey = copyObjectRequest.getDestinationKey();
        String destinationObjectVersion = MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED.equals(destinationBucketName) ? UUID.randomUUID().toString() : null;
        String destinationObjectKeyVersion = destinationObjectKey + (destinationObjectVersion != null ? destinationObjectVersion : "");

        ObjectMetadata objectMetadata = copyObjectRequest.getNewObjectMetadata();
        MockS3Object mockDestinationS3Object = new MockS3Object();
        mockDestinationS3Object.setKey(destinationObjectKey);
        mockDestinationS3Object.setVersion(destinationObjectVersion);
        mockDestinationS3Object.setData(Arrays.copyOf(mockSourceS3Object.getData(), mockSourceS3Object.getData().length));
        mockDestinationS3Object.setObjectMetadata(objectMetadata);

        MockS3Bucket mockDestinationS3Bucket = getOrCreateBucket(destinationBucketName);
        mockDestinationS3Bucket.getObjects().put(destinationObjectKey, mockDestinationS3Object);
        mockDestinationS3Bucket.getVersions().put(destinationObjectKeyVersion, mockDestinationS3Object);
    }

    return copy;
}