com.amazonaws.services.s3.transfer.Copy Java Examples

private Copy submitCopyJob(CopyJobRequest copyJob) {
  CopyObjectRequest copyObjectRequest = copyJob.getCopyObjectRequest();
  LOG
      .info("Copying object from '{}/{}' to '{}/{}'", copyObjectRequest.getSourceBucketName(),
          copyObjectRequest.getSourceKey(), copyObjectRequest.getDestinationBucketName(),
          copyObjectRequest.getDestinationKey());
  return transferManager.copy(copyObjectRequest, srcClient, copyJob.getTransferStateChangeListener());
}
 

@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}
 

@Test
public void copyDefaultCopierOptions() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);

  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  ArgumentCaptor<CopyObjectRequest> argument = ArgumentCaptor.forClass(CopyObjectRequest.class);
  verify(mockedTransferManager).copy(argument.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
  CopyObjectRequest copyObjectRequest = argument.getValue();
  assertNull(copyObjectRequest.getNewObjectMetadata());
}
 

public static void copyObjectSimple(String from_bucket, String from_key,
                                    String to_bucket, String to_key) {
    // snippet-start:[s3.java1.s3_xfer_mgr_copy.copy_object]
    System.out.println("Copying s3 object: " + from_key);
    System.out.println("      from bucket: " + from_bucket);
    System.out.println("     to s3 object: " + to_key);
    System.out.println("        in bucket: " + to_bucket);

    TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
    try {
        Copy xfer = xfer_mgr.copy(from_bucket, from_key, to_bucket, to_key);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
    // snippet-end:[s3.java1.s3_xfer_mgr_copy.copy_object]
}
 

private List<CopyJobRequest> submitAndGatherCopyJobs(List<CopyJobRequest> copyJobsToSubmit) {
  List<CopyJob> submittedCopyJobs = new ArrayList<>();
  for (CopyJobRequest copyJobRequest : copyJobsToSubmit) {
    Copy copy = submitCopyJob(copyJobRequest);
    CopyJob newCopyJob = new CopyJob(copy, copyJobRequest);
    submittedCopyJobs.add(newCopyJob);
  }
  return gatherCopyJobs(submittedCopyJobs);
}
 

@Test
public void copyCheckTransferManagerIsShutdownWhenMaxRetriesExceeded() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    verify(mockedTransferManager, Mockito.times(3))
        .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class));
    assertThat(e.getMessage(), is("1 job(s) failed the maximum number of copy attempts, 3"));
  }
}
 

@Test
public void copyServerSideEncryption() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  Map<String, Object> copierOptions = new HashMap<>();
  copierOptions.put(S3S3CopierOptions.Keys.S3_SERVER_SIDE_ENCRYPTION.keyName(), "true");
  S3S3CopierOptions customOptions = new S3S3CopierOptions(copierOptions);

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(customOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);

  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, customOptions);
  s3s3Copier.copy();
  ArgumentCaptor<CopyObjectRequest> argument = ArgumentCaptor.forClass(CopyObjectRequest.class);
  verify(mockedTransferManager).copy(argument.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
  CopyObjectRequest copyObjectRequest = argument.getValue();
  assertThat(copyObjectRequest.getNewObjectMetadata().getSSEAlgorithm(),
      is(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION));
}
 

@Test
public void copyCannedAcl() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  Map<String, Object> copierOptions = new HashMap<>();
  copierOptions
      .put(S3S3CopierOptions.Keys.CANNED_ACL.keyName(), CannedAccessControlList.BucketOwnerFullControl.toString());
  S3S3CopierOptions customOptions = new S3S3CopierOptions(copierOptions);

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(customOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);

  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, customOptions);
  s3s3Copier.copy();
  ArgumentCaptor<CopyObjectRequest> argument = ArgumentCaptor.forClass(CopyObjectRequest.class);
  verify(mockedTransferManager).copy(argument.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
  CopyObjectRequest copyObjectRequest = argument.getValue();
  assertThat(copyObjectRequest.getCannedAccessControlList(), is(CannedAccessControlList.BucketOwnerFullControl));
}
 

@Test
public void copySafelyShutDownTransferWhenRetryFails() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenThrow(new AmazonClientException("S3 error"));
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("Exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getMessage(), is("Error in S3S3Copier:"));
    assertThat(e.getCause().getMessage(), startsWith("S3 error"));
  }
}
 

private void copyFile(String srcKey, String dstKey) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("copyFile " + srcKey + " -> " + dstKey);
  }

  ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
  final ObjectMetadata dstom = srcom.clone();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    dstom.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
  copyObjectRequest.setCannedAccessControlList(cannedACL);
  copyObjectRequest.setNewObjectMetadata(dstom);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Copy copy = transfers.copy(copyObjectRequest);
  copy.addProgressListener(progressListener);
  try {
    copy.waitForCopyResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }
}
 

private void copyFile(String srcKey, String dstKey) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("copyFile " + srcKey + " -> " + dstKey);
  }

  ObjectMetadata srcom = s3.getObjectMetadata(bucket, srcKey);
  final ObjectMetadata dstom = srcom.clone();
  if (StringUtils.isNotBlank(serverSideEncryptionAlgorithm)) {
    dstom.setServerSideEncryption(serverSideEncryptionAlgorithm);
  }
  CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucket, srcKey, bucket, dstKey);
  copyObjectRequest.setCannedAccessControlList(cannedACL);
  copyObjectRequest.setNewObjectMetadata(dstom);

  ProgressListener progressListener = new ProgressListener() {
    public void progressChanged(ProgressEvent progressEvent) {
      switch (progressEvent.getEventCode()) {
        case ProgressEvent.PART_COMPLETED_EVENT_CODE:
          statistics.incrementWriteOps(1);
          break;
        default:
          break;
      }
    }
  };

  Copy copy = transfers.copy(copyObjectRequest);
  copy.addProgressListener(progressListener);
  try {
    copy.waitForCopyResult();
    statistics.incrementWriteOps(1);
  } catch (InterruptedException e) {
    throw new IOException("Got interrupted, cancelling");
  }
}
 

/**
 * {@inheritDoc} <p/> <p> This implementation simulates a copyFile operation. </p> <p> This method copies files in-memory. </p> <p> The result {@link Copy}
 * has the following properties: <dl> <p/> <dt>description</dt> <dd>"MockTransfer"</dd> <p/> <dt>state</dt> <dd>{@link TransferState#Completed}</dd> <p/>
 * <dt>transferProgress.totalBytesToTransfer</dt> <dd>1024</dd> <p/> <dt>transferProgress.updateProgress</dt> <dd>1024</dd> <p/> </dl> <p/> All other
 * properties are set as default. </p> <p> This operation takes the following hints when suffixed in copyObjectRequest.sourceKey: <dl> <p/>
 * <dt>MOCK_S3_FILE_NAME_SERVICE_EXCEPTION</dt> <dd>Throws a AmazonServiceException</dd> <p/> </dl> </p>
 */
@Override
public Copy copyFile(final CopyObjectRequest copyObjectRequest, TransferManager transferManager)
{
    LOGGER.debug(
        "copyFile(): copyObjectRequest.getSourceBucketName() = " + copyObjectRequest.getSourceBucketName() + ", copyObjectRequest.getSourceKey() = " +
            copyObjectRequest.getSourceKey() + ", copyObjectRequest.getDestinationBucketName() = " + copyObjectRequest.getDestinationBucketName() +
            ", copyObjectRequest.getDestinationKey() = " + copyObjectRequest.getDestinationKey());

    if (copyObjectRequest.getSourceKey().endsWith(MOCK_S3_FILE_NAME_SERVICE_EXCEPTION))
    {
        throw new AmazonServiceException(null);
    }

    String sourceBucketName = copyObjectRequest.getSourceBucketName();
    String sourceKey = copyObjectRequest.getSourceKey();

    MockS3Bucket mockSourceS3Bucket = getOrCreateBucket(sourceBucketName);
    MockS3Object mockSourceS3Object = mockSourceS3Bucket.getObjects().get(sourceKey);

    if (mockSourceS3Object == null)
    {
        AmazonServiceException amazonServiceException = new AmazonServiceException(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        amazonServiceException.setErrorCode(S3Operations.ERROR_CODE_NO_SUCH_KEY);
        throw amazonServiceException;
    }

    // Set the result CopyImpl and TransferProgress.
    TransferProgress transferProgress = new TransferProgress();
    transferProgress.setTotalBytesToTransfer(mockSourceS3Object.getObjectMetadata().getContentLength());
    transferProgress.updateProgress(mockSourceS3Object.getObjectMetadata().getContentLength());
    CopyImpl copy = new CopyImpl(MOCK_TRANSFER_DESCRIPTION, transferProgress, null, null);
    copy.setState(TransferState.Completed);

    // If an invalid KMS Id was passed in, mark the transfer as failed and return an exception via the transfer monitor.
    if (copyObjectRequest.getSSEAwsKeyManagementParams() != null)
    {
        final String kmsId = copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId();
        if (kmsId.startsWith(MOCK_KMS_ID_FAILED_TRANSFER))
        {
            copy.setState(TransferState.Failed);
            copy.setMonitor(new TransferMonitor()
            {
                @Override
                public Future<?> getFuture()
                {
                    if (!kmsId.equals(MOCK_KMS_ID_FAILED_TRANSFER_NO_EXCEPTION))
                    {
                        throw new AmazonServiceException("Key '" + copyObjectRequest.getSSEAwsKeyManagementParams().getAwsKmsKeyId() +
                            "' does not exist (Service: Amazon S3; Status Code: 400; Error Code: KMS.NotFoundException; Request ID: 1234567890123456)");
                    }

                    // We don't want an exception to be thrown so return a basic future that won't throw an exception.
                    BasicFuture<?> future = new BasicFuture<Void>(null);
                    future.completed(null);
                    return future;
                }

                @Override
                public boolean isDone()
                {
                    return true;
                }
            });
        }
        else if (kmsId.startsWith(MOCK_KMS_ID_CANCELED_TRANSFER))
        {
            // If the KMS indicates a cancelled transfer, just update the state to canceled.
            copy.setState(TransferState.Canceled);
        }
    }

    // If copy operation is marked as completed, perform the actual file copy in memory.
    if (copy.getState().equals(TransferState.Completed))
    {
        String destinationBucketName = copyObjectRequest.getDestinationBucketName();
        String destinationObjectKey = copyObjectRequest.getDestinationKey();
        String destinationObjectVersion = MOCK_S3_BUCKET_NAME_VERSIONING_ENABLED.equals(destinationBucketName) ? UUID.randomUUID().toString() : null;
        String destinationObjectKeyVersion = destinationObjectKey + (destinationObjectVersion != null ? destinationObjectVersion : "");

        ObjectMetadata objectMetadata = copyObjectRequest.getNewObjectMetadata();
        MockS3Object mockDestinationS3Object = new MockS3Object();
        mockDestinationS3Object.setKey(destinationObjectKey);
        mockDestinationS3Object.setVersion(destinationObjectVersion);
        mockDestinationS3Object.setData(Arrays.copyOf(mockSourceS3Object.getData(), mockSourceS3Object.getData().length));
        mockDestinationS3Object.setObjectMetadata(objectMetadata);

        MockS3Bucket mockDestinationS3Bucket = getOrCreateBucket(destinationBucketName);
        mockDestinationS3Bucket.getObjects().put(destinationObjectKey, mockDestinationS3Object);
        mockDestinationS3Bucket.getVersions().put(destinationObjectKeyVersion, mockDestinationS3Object);
    }

    return copy;
}
 

public CopyJob(Copy copy, CopyJobRequest copyJobRequest) {
  this.copy = copy;
  this.copyJobRequest = copyJobRequest;
}
 

@Test
public void testPerformTransferAssertHandleFailedWithAmazonClientException() throws Exception
{
    S3Operations originalS3Operations = (S3Operations) ReflectionTestUtils.getField(s3Dao, "s3Operations");
    S3Operations mockS3Operations = mock(S3Operations.class);
    ReflectionTestUtils.setField(s3Dao, "s3Operations", mockS3Operations);

    // Shorten the sleep interval for faster tests
    long originalSleepIntervalsMillis = (long) ReflectionTestUtils.getField(s3Dao, "sleepIntervalsMillis");
    ReflectionTestUtils.setField(s3Dao, "sleepIntervalsMillis", 1l);

    try
    {
        S3FileCopyRequestParamsDto s3FileCopyRequestParamsDto = new S3FileCopyRequestParamsDto();
        s3FileCopyRequestParamsDto.setSourceBucketName("sourceBucketName");
        s3FileCopyRequestParamsDto.setSourceObjectKey("sourceObjectKey");
        s3FileCopyRequestParamsDto.setTargetBucketName("targetBucketName");
        s3FileCopyRequestParamsDto.setTargetObjectKey("targetObjectKey");
        s3FileCopyRequestParamsDto.setKmsKeyId("kmsKeyId");

        when(mockS3Operations.copyFile(any(), any())).then(new Answer<Copy>()
        {
            @Override
            public Copy answer(InvocationOnMock invocation) throws Throwable
            {
                Copy mockTransfer = mock(Copy.class);

                when(mockTransfer.getProgress()).thenReturn(new TransferProgress());
                when(mockTransfer.getState()).thenReturn(TransferState.Failed);
                when(mockTransfer.isDone()).thenReturn(true);
                when(mockTransfer.waitForException()).thenReturn(new AmazonClientException("message"));
                return mockTransfer;
            }
        });

        try
        {
            s3Dao.copyFile(s3FileCopyRequestParamsDto);
            fail();
        }
        catch (Exception e)
        {
            assertEquals(AmazonClientException.class, e.getClass());
            assertEquals("message", e.getMessage());
        }
    }
    finally
    {
        ReflectionTestUtils.setField(s3Dao, "s3Operations", originalS3Operations);
        ReflectionTestUtils.setField(s3Dao, "sleepIntervalsMillis", originalSleepIntervalsMillis);
    }
}
 

@Override
public Copy copyFile(CopyObjectRequest copyObjectRequest, TransferManager transferManager)
{
    return transferManager.copy(copyObjectRequest);
}
 

@Override
public String run() throws Exception {
	final String fromBucket = this.step.getFromBucket();
	final String toBucket = this.step.getToBucket();
	final String fromPath = this.step.getFromPath();
	final String toPath = this.step.getToPath();
	final String kmsId = this.step.getKmsId();
	final Map<String, String> metadatas = new HashMap<>();
	final CannedAccessControlList acl = this.step.getAcl();
	final String cacheControl = this.step.getCacheControl();
	final String contentType = this.step.getContentType();
	final String sseAlgorithm = this.step.getSseAlgorithm();
	final S3ClientOptions s3ClientOptions = this.step.createS3ClientOptions();
	final EnvVars envVars = this.getContext().get(EnvVars.class);

	if (this.step.getMetadatas() != null && this.step.getMetadatas().length != 0) {
		for (String metadata : this.step.getMetadatas()) {
			if (metadata.split(":").length == 2) {
				metadatas.put(metadata.split(":")[0], metadata.split(":")[1]);
			}
		}
	}

	Preconditions.checkArgument(fromBucket != null && !fromBucket.isEmpty(), "From bucket must not be null or empty");
	Preconditions.checkArgument(fromPath != null && !fromPath.isEmpty(), "From path must not be null or empty");
	Preconditions.checkArgument(toBucket != null && !toBucket.isEmpty(), "To bucket must not be null or empty");
	Preconditions.checkArgument(toPath != null && !toPath.isEmpty(), "To path must not be null or empty");

	TaskListener listener = Execution.this.getContext().get(TaskListener.class);
	listener.getLogger().format("Copying s3://%s/%s to s3://%s/%s%n", fromBucket, fromPath, toBucket, toPath);

	CopyObjectRequest request = new CopyObjectRequest(fromBucket, fromPath, toBucket, toPath);

	// Add metadata
	if (metadatas.size() > 0 || (cacheControl != null && !cacheControl.isEmpty()) || (contentType != null && !contentType.isEmpty()) || (sseAlgorithm != null && !sseAlgorithm.isEmpty())) {
		ObjectMetadata metas = new ObjectMetadata();
		if (metadatas.size() > 0) {
			metas.setUserMetadata(metadatas);
		}
		if (cacheControl != null && !cacheControl.isEmpty()) {
			metas.setCacheControl(cacheControl);
		}
		if (contentType != null && !contentType.isEmpty()) {
			metas.setContentType(contentType);
		}
		if (sseAlgorithm != null && !sseAlgorithm.isEmpty()) {
			metas.setSSEAlgorithm(sseAlgorithm);
		}
		request.withNewObjectMetadata(metas);
	}

	// Add acl
	if (acl != null) {
		request.withCannedAccessControlList(acl);
	}

	// Add kms
	if (kmsId != null && !kmsId.isEmpty()) {
		listener.getLogger().format("Using KMS: %s%n", kmsId);
		request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsId));
	}

	TransferManager mgr = TransferManagerBuilder.standard()
			.withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars))
			.build();
	try {
		final Copy copy = mgr.copy(request);
		copy.addProgressListener((ProgressListener) progressEvent -> {
			if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) {
				listener.getLogger().println("Finished: " + copy.getDescription());
			}
		});
		copy.waitForCompletion();
	}
	finally{
		mgr.shutdownNow();
	}

	listener.getLogger().println("Copy complete");
	return String.format("s3://%s/%s", toBucket, toPath);
}
 

@Test
public void copyRetryOnlyFailedCopyJobs() throws InterruptedException {
  String sourceKey1 = "bar/data1";
  String sourceKey2 = "bar/data2";
  client.putObject("source", sourceKey1, inputData);
  client.putObject("source", sourceKey2, inputData);
  Path sourceBaseLocation = new Path("s3://source/bar/");
  Path replicaLocation = new Path("s3://target/foo/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager
      .copy(any(CopyObjectRequest.class), any(AmazonS3.class), any(TransferStateChangeListener.class)))
          .thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  transferProgress.setTotalBytesToTransfer(7);
  when(copy.getProgress()).thenReturn(transferProgress);
  doThrow(new AmazonClientException("cause")).doNothing().when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    Metrics metrics = s3s3Copier.copy();
    ArgumentCaptor<CopyObjectRequest> captor = ArgumentCaptor.forClass(CopyObjectRequest.class);
    verify(mockedTransferManager, Mockito.times(3))
        .copy(captor.capture(), any(AmazonS3.class), any(TransferStateChangeListener.class));
    List<CopyObjectRequest> capturedCopyRequests = captor.getAllValues();
    assertThat(capturedCopyRequests.get(0).getSourceKey(), is(sourceKey1));
    assertThat(capturedCopyRequests.get(1).getSourceKey(), is(sourceKey2));
    assertThat(capturedCopyRequests.get(2).getSourceKey(), is(sourceKey1));
    verify(mockedTransferManager).shutdownNow();
    verifyNoMoreInteractions(mockedTransferManager);
    assertThat(metrics.getBytesReplicated(), is(14L));
    assertThat(metrics.getMetrics().get(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name()), is(14L));
  } catch (CircusTrainException e) {
    fail("Exception should not have been thrown");
  }
}
 

public Copy getCopy() {
  return copy;
}
 

/**
 * Schedules a new transfer to copy data from one Amazon S3 location to another Amazon S3 location.
 *
 * @param copyObjectRequest the request containing all the parameters for the copy
 * @param transferManager the transfer manager implementation to use
 */
public Copy copyFile(CopyObjectRequest copyObjectRequest, TransferManager transferManager);