com.amazonaws.services.s3.transfer.TransferManagerBuilder Java Examples
The following examples show how to use
com.amazonaws.services.s3.transfer.TransferManagerBuilder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3UtilProgram.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String localFolder = params.getRequired("localFolder"); final String s3filePrefix = params.get("s3filePrefix", ""); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : objectSummary -> keyPredicate.test(objectSummary.getKey()); try { tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
Example #2
Source File: S3UtilProgram.java From flink with Apache License 2.0 | 6 votes |
private static void downloadByFullPathAndFileNamePrefix(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3prefix = params.getRequired("s3prefix"); final String localFolder = params.getRequired("localFolder"); final String s3filePrefix = params.get("s3filePrefix", ""); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); Predicate<String> keyPredicate = getKeyFilterByFileNamePrefix(s3filePrefix); KeyFilter keyFilter = s3filePrefix.isEmpty() ? KeyFilter.INCLUDE_ALL : objectSummary -> keyPredicate.test(objectSummary.getKey()); try { tx.downloadDirectory(bucket, s3prefix, new File(localFolder), keyFilter).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
Example #3
Source File: CopyMapper.java From circus-train with Apache License 2.0 | 6 votes |
/** * Implementation of the Mapper::setup() method. This extracts the S3MapReduceCp options specified in the Job's * configuration, to set up the Job. * * @param context Mapper's context. * @throws IOException On IO failure. * @throws InterruptedException If the job is interrupted. */ @Override public void setup(Context context) throws IOException, InterruptedException { conf = new S3MapReduceCpConfiguration(context.getConfiguration()); ignoreFailures = conf.getBoolean(ConfigurationVariable.IGNORE_FAILURES); targetFinalPath = new Path(conf.get(S3MapReduceCpConstants.CONF_LABEL_TARGET_FINAL_PATH)); AwsS3ClientFactory awsS3ClientFactory = new AwsS3ClientFactory(); transferManager = TransferManagerBuilder .standard() .withMinimumUploadPartSize(conf.getLong(ConfigurationVariable.MINIMUM_UPLOAD_PART_SIZE)) .withMultipartUploadThreshold(conf.getLong(ConfigurationVariable.MULTIPART_UPLOAD_THRESHOLD)) .withS3Client(awsS3ClientFactory.newInstance(conf)) .withShutDownThreadPools(true) .withExecutorFactory(new ExecutorFactory() { @Override public ExecutorService newExecutor() { return Executors.newFixedThreadPool(conf.getInt(ConfigurationVariable.NUMBER_OF_UPLOAD_WORKERS)); } }) .build(); }
Example #4
Source File: XferMgrUpload.java From aws-doc-sdk-examples with Apache License 2.0 | 6 votes |
public static void uploadDir(String dir_path, String bucket_name, String key_prefix, boolean recursive, boolean pause) { System.out.println("directory: " + dir_path + (recursive ? " (recursive)" : "") + (pause ? " (pause)" : "")); // snippet-start:[s3.java1.s3_xfer_mgr_upload.directory] TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name, key_prefix, new File(dir_path), recursive); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); // snippet-end:[s3.java1.s3_xfer_mgr_upload.directory] }
Example #5
Source File: XferMgrCopy.java From aws-doc-sdk-examples with Apache License 2.0 | 6 votes |
public static void copyObjectSimple(String from_bucket, String from_key, String to_bucket, String to_key) { // snippet-start:[s3.java1.s3_xfer_mgr_copy.copy_object] System.out.println("Copying s3 object: " + from_key); System.out.println(" from bucket: " + from_bucket); System.out.println(" to s3 object: " + to_key); System.out.println(" in bucket: " + to_bucket); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { Copy xfer = xfer_mgr.copy(from_bucket, from_key, to_bucket, to_key); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); // snippet-end:[s3.java1.s3_xfer_mgr_copy.copy_object] }
Example #6
Source File: XferMgrDownload.java From aws-doc-sdk-examples with Apache License 2.0 | 6 votes |
public static void downloadFile(String bucket_name, String key_name, String file_path, boolean pause) { System.out.println("Downloading to file: " + file_path + (pause ? " (pause)" : "")); // snippet-start:[s3.java1.s3_xfer_mgr_download.single] File f = new File(file_path); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { Download xfer = xfer_mgr.download(bucket_name, key_name, f); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); // snippet-end:[s3.java1.s3_xfer_mgr_download.single] }
Example #7
Source File: XferMgrDownload.java From aws-doc-sdk-examples with Apache License 2.0 | 6 votes |
public static void downloadDir(String bucket_name, String key_prefix, String dir_path, boolean pause) { System.out.println("downloading to directory: " + dir_path + (pause ? " (pause)" : "")); // snippet-start:[s3.java1.s3_xfer_mgr_download.directory] TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileDownload xfer = xfer_mgr.downloadDirectory( bucket_name, key_prefix, new File(dir_path)); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); // snippet-end:[s3.java1.s3_xfer_mgr_download.directory] }
Example #8
Source File: PrestoS3FileSystem.java From presto with Apache License 2.0 | 5 votes |
public PrestoS3OutputStream( AmazonS3 s3, String bucket, String key, File tempFile, boolean sseEnabled, PrestoS3SseType sseType, String sseKmsKeyId, long multiPartUploadMinFileSize, long multiPartUploadMinPartSize, PrestoS3AclType aclType, boolean requesterPaysEnabled, PrestoS3StorageClass s3StorageClass) throws IOException { super(new BufferedOutputStream(new FileOutputStream(requireNonNull(tempFile, "tempFile is null")))); transferManager = TransferManagerBuilder.standard() .withS3Client(requireNonNull(s3, "s3 is null")) .withMinimumUploadPartSize(multiPartUploadMinPartSize) .withMultipartUploadThreshold(multiPartUploadMinFileSize).build(); requireNonNull(aclType, "aclType is null"); requireNonNull(s3StorageClass, "s3StorageClass is null"); this.aclType = aclType.getCannedACL(); this.bucket = requireNonNull(bucket, "bucket is null"); this.key = requireNonNull(key, "key is null"); this.tempFile = tempFile; this.sseEnabled = sseEnabled; this.sseType = requireNonNull(sseType, "sseType is null"); this.sseKmsKeyId = sseKmsKeyId; this.requesterPaysEnabled = requesterPaysEnabled; this.s3StorageClass = s3StorageClass.getS3StorageClass(); log.debug("OutputStream for key '%s' using file: %s", key, tempFile); }
Example #9
Source File: MultipartUploadLiveTest.java From tutorials with MIT License | 5 votes |
@Before public void setup() { amazonS3 = mock(AmazonS3.class); tm = TransferManagerBuilder .standard() .withS3Client(amazonS3) .withMultipartUploadThreshold((long) (5 * 1024 * 1025)) .withExecutorFactory(() -> Executors.newFixedThreadPool(5)) .build(); progressListener = progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred()); }
Example #10
Source File: MultipartUpload.java From tutorials with MIT License | 5 votes |
public static void main(String[] args) throws Exception { String existingBucketName = "baeldung-bucket"; String keyName = "my-picture.jpg"; String filePath = "documents/my-picture.jpg"; AmazonS3 amazonS3 = AmazonS3ClientBuilder .standard() .withCredentials(new DefaultAWSCredentialsProviderChain()) .withRegion(Regions.DEFAULT_REGION) .build(); int maxUploadThreads = 5; TransferManager tm = TransferManagerBuilder .standard() .withS3Client(amazonS3) .withMultipartUploadThreshold((long) (5 * 1024 * 1024)) .withExecutorFactory(() -> Executors.newFixedThreadPool(maxUploadThreads)) .build(); ProgressListener progressListener = progressEvent -> System.out.println("Transferred bytes: " + progressEvent.getBytesTransferred()); PutObjectRequest request = new PutObjectRequest(existingBucketName, keyName, new File(filePath)); request.setGeneralProgressListener(progressListener); Upload upload = tm.upload(request); try { upload.waitForCompletion(); System.out.println("Upload complete."); } catch (AmazonClientException e) { System.out.println("Error occurred while uploading file"); e.printStackTrace(); } }
Example #11
Source File: AwsS3ObjectStorage.java From james-project with Apache License 2.0 | 5 votes |
private static TransferManager getTransferManager(AmazonS3 s3Client, ExecutorService executorService) { return TransferManagerBuilder .standard() .withS3Client(s3Client) .withMultipartUploadThreshold(MULTIPART_UPLOAD_THRESHOLD.getValue()) .withExecutorFactory(() -> executorService) .withShutDownThreadPools(DO_NOT_SHUTDOWN_THREAD_POOL) .build(); }
Example #12
Source File: S3Manager.java From datacollector with Apache License 2.0 | 5 votes |
S3Manager(EmrClusterConfig pipelineEmrConfigs, String pipelineId, String uniquePrefix, List<File> filesToUpload) { this.pipelineEmrConfigs = pipelineEmrConfigs; this.pipelineId = pipelineId; this.uniquePrefix = uniquePrefix; this.filesToUpload = filesToUpload; s3Client = getS3Client(); int uploadThreads = Integer.parseInt(System.getProperty("sdc.emr.s3.uploadThreads", "5")); s3TransferManager = TransferManagerBuilder.standard() .withS3Client(s3Client) .withMultipartUploadThreshold(10L*1024*1024) .withMinimumUploadPartSize(10L*1024*1024) .withExecutorFactory(() -> Executors.newFixedThreadPool(uploadThreads)) .withShutDownThreadPools(true) .build(); }
Example #13
Source File: S3FileManagerImpl.java From entrada with GNU General Public License v3.0 | 5 votes |
public S3FileManagerImpl(AmazonS3 amazonS3, @Value("${aws.upload.parallelism}") int parallelism, @Value("${aws.upload.multipart.mb.size:5}") int multipartSize) { this.amazonS3 = amazonS3; this.transferManager = TransferManagerBuilder .standard() .withS3Client(amazonS3) .withMultipartUploadThreshold(multipartSize * 1024L * 1024L) .withExecutorFactory(() -> Executors.newFixedThreadPool(parallelism)) .build(); }
Example #14
Source File: TransferManagerUploader.java From nexus-public with Eclipse Public License 1.0 | 5 votes |
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) { try { TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3).build(); transferManager.upload(bucket, key, contents, new ObjectMetadata()) .waitForCompletion(); } catch (InterruptedException e) { throw new BlobStoreException("error uploading blob", e, null); } }
Example #15
Source File: XferMgrUpload.java From aws-doc-sdk-examples with Apache License 2.0 | 5 votes |
public static void uploadFile(String file_path, String bucket_name, String key_prefix, boolean pause) { System.out.println("file: " + file_path + (pause ? " (pause)" : "")); String key_name = null; if (key_prefix != null) { key_name = key_prefix + '/' + file_path; } else { key_name = file_path; } // snippet-start:[s3.java1.s3_xfer_mgr_upload.single] File f = new File(file_path); TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { Upload xfer = xfer_mgr.upload(bucket_name, key_name, f); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); // snippet-end:[s3.java1.s3_xfer_mgr_upload.single] }
Example #16
Source File: XferMgrUpload.java From aws-doc-sdk-examples with Apache License 2.0 | 5 votes |
public static void uploadFileList(String[] file_paths, String bucket_name, String key_prefix, boolean pause) { System.out.println("file list: " + Arrays.toString(file_paths) + (pause ? " (pause)" : "")); // convert the file paths to a list of File objects (required by the // uploadFileList method) // snippet-start:[s3.java1.s3_xfer_mgr_upload.list_of_files] ArrayList<File> files = new ArrayList<File>(); for (String path : file_paths) { files.add(new File(path)); } TransferManager xfer_mgr = TransferManagerBuilder.standard().build(); try { MultipleFileUpload xfer = xfer_mgr.uploadFileList(bucket_name, key_prefix, new File("."), files); // loop with Transfer.isDone() XferMgrProgress.showTransferProgress(xfer); // or block with Transfer.waitForCompletion() XferMgrProgress.waitForCompletion(xfer); } catch (AmazonServiceException e) { System.err.println(e.getErrorMessage()); System.exit(1); } xfer_mgr.shutdownNow(); // snippet-end:[s3.java1.s3_xfer_mgr_upload.list_of_files] }
Example #17
Source File: S3UploadStepTransferManagerIntegrationTest.java From pipeline-aws-plugin with Apache License 2.0 | 5 votes |
@Before public void setupSdk() throws Exception { PowerMockito.mockStatic(AWSClientFactory.class); AmazonS3 amazonS3 = PowerMockito.mock(AmazonS3.class); PowerMockito.when(AWSClientFactory.create(Mockito.any(AwsSyncClientBuilder.class), Mockito.any(StepContext.class))) .thenReturn(amazonS3); PowerMockito.mockStatic(TransferManagerBuilder.class); transferManager = Mockito.mock(TransferManager.class); TransferManagerBuilder transferManagerBuilder = PowerMockito.mock(TransferManagerBuilder.class); PowerMockito.when(TransferManagerBuilder.standard()).thenReturn(transferManagerBuilder); PowerMockito.when(transferManagerBuilder.withS3Client(Mockito.any(AmazonS3.class))).thenReturn(transferManagerBuilder); PowerMockito.when(transferManagerBuilder.build()).thenReturn(transferManager); }
Example #18
Source File: S3DownloadStep.java From pipeline-aws-plugin with Apache License 2.0 | 5 votes |
@Override public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException { TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars)) .build(); if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) { try { final MultipleFileDownload fileDownload = mgr.downloadDirectory(this.bucket, this.path, localFile); fileDownload.waitForCompletion(); RemoteDownloader.this.taskListener.getLogger().println("Finished: " + fileDownload.getDescription()); } finally { mgr.shutdownNow(); } return null; } else { try { final Download download = mgr.download(this.bucket, this.path, localFile); download.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { RemoteDownloader.this.taskListener.getLogger().println("Finished: " + download.getDescription()); } }); download.waitForCompletion(); } finally { mgr.shutdownNow(); } return null; } }
Example #19
Source File: TablonController.java From spring-cloud-aws-sample with Apache License 2.0 | 5 votes |
@RequestMapping("/anuncio/nuevo") public String nuevoAnuncio(Model model, @RequestParam String nombre, @RequestParam String asunto, @RequestParam String comentario, @RequestParam String filename, @RequestParam MultipartFile file) { if (!file.isEmpty()) { try { ObjectMetadata objectMetadata = new ObjectMetadata(); objectMetadata.setContentType(file.getContentType()); TransferManager transferManager = TransferManagerBuilder.defaultTransferManager(); transferManager.upload(bucket, filename, file.getInputStream(), objectMetadata); } catch (Exception e) { model.addAttribute("message", "You failed to upload " + filename + " => " + e.getMessage()); return "error"; } } else { model.addAttribute("message", "You failed to upload " + filename + " because the file was empty."); return "error"; } Anuncio anuncio = new Anuncio(nombre, asunto, comentario); anuncio.setFoto(s3.getUrl(bucket, filename)); repository.save(anuncio); return "anuncio_guardado"; }
Example #20
Source File: TransferManagerFactory.java From circus-train with Apache License 2.0 | 5 votes |
public TransferManager newInstance(AmazonS3 targetS3Client, S3S3CopierOptions s3s3CopierOptions) { LOG .debug("Initializing transfer manager with {} threads.", s3s3CopierOptions.getMaxThreadPoolSize()); return TransferManagerBuilder.standard() .withMultipartCopyThreshold(s3s3CopierOptions.getMultipartCopyThreshold()) .withMultipartCopyPartSize(s3s3CopierOptions.getMultipartCopyPartSize()) .withExecutorFactory(() -> Executors.newFixedThreadPool(s3s3CopierOptions.getMaxThreadPoolSize())) .withS3Client(targetS3Client) .build(); }
Example #21
Source File: S3UtilProgram.java From flink with Apache License 2.0 | 5 votes |
private static void downloadFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); final String localFile = params.getRequired("localFile"); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); try { tx.download(bucket, s3file, new File(localFile)).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
Example #22
Source File: S3UtilProgram.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void downloadFile(ParameterTool params) { final String bucket = params.getRequired("bucket"); final String s3file = params.getRequired("s3file"); final String localFile = params.getRequired("localFile"); TransferManager tx = TransferManagerBuilder.defaultTransferManager(); try { tx.download(bucket, s3file, new File(localFile)).waitForCompletion(); } catch (InterruptedException e) { System.out.println("Transfer interrupted"); } finally { tx.shutdownNow(); } }
Example #23
Source File: S3Downloader.java From aws-codebuild-jenkins-plugin with Apache License 2.0 | 4 votes |
public S3Downloader(AmazonS3Client s3Client) { this.s3Client = s3Client; transferManager = TransferManagerBuilder.standard().withS3Client(s3Client).build(); }
Example #24
Source File: S3UploadStep.java From pipeline-aws-plugin with Apache License 2.0 | 4 votes |
@Override public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException { TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars)) .build(); final MultipleFileUpload fileUpload; ObjectMetadataProvider metadatasProvider = (file, meta) -> { if (meta != null) { if (RemoteListUploader.this.metadatas != null && RemoteListUploader.this.metadatas.size() > 0) { meta.setUserMetadata(RemoteListUploader.this.metadatas); } if (RemoteListUploader.this.acl != null) { meta.setHeader(Headers.S3_CANNED_ACL, RemoteListUploader.this.acl); } if (RemoteListUploader.this.cacheControl != null && !RemoteListUploader.this.cacheControl.isEmpty()) { meta.setCacheControl(RemoteListUploader.this.cacheControl); } if (RemoteListUploader.this.contentEncoding != null && !RemoteListUploader.this.contentEncoding.isEmpty()) { meta.setContentEncoding(RemoteListUploader.this.contentEncoding); } if (RemoteListUploader.this.contentType != null && !RemoteListUploader.this.contentType.isEmpty()) { meta.setContentType(RemoteListUploader.this.contentType); } if (RemoteListUploader.this.sseAlgorithm != null && !RemoteListUploader.this.sseAlgorithm.isEmpty()) { meta.setSSEAlgorithm(RemoteListUploader.this.sseAlgorithm); } if (RemoteListUploader.this.kmsId != null && !RemoteListUploader.this.kmsId.isEmpty()) { final SSEAwsKeyManagementParams sseAwsKeyManagementParams = new SSEAwsKeyManagementParams(RemoteListUploader.this.kmsId); meta.setSSEAlgorithm(sseAwsKeyManagementParams.getAwsKmsKeyId()); meta.setHeader( Headers.SERVER_SIDE_ENCRYPTION_AWS_KMS_KEYID, sseAwsKeyManagementParams.getAwsKmsKeyId() ); } } }; ObjectTaggingProvider objectTaggingProvider =(uploadContext) -> { List<Tag> tagList = new ArrayList<Tag>(); //add tags if(tags != null){ for (Map.Entry<String, String> entry : tags.entrySet()) { Tag tag = new Tag(entry.getKey(), entry.getValue()); tagList.add(tag); } } return new ObjectTagging(tagList); }; try { fileUpload = mgr.uploadFileList(this.bucket, this.path, localFile, this.fileList, metadatasProvider, objectTaggingProvider); for (final Upload upload : fileUpload.getSubTransfers()) { upload.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { RemoteListUploader.this.taskListener.getLogger().println("Finished: " + upload.getDescription()); } }); } fileUpload.waitForCompletion(); } finally { mgr.shutdownNow(); } return null; }
Example #25
Source File: S3CopyStep.java From pipeline-aws-plugin with Apache License 2.0 | 4 votes |
@Override public String run() throws Exception { final String fromBucket = this.step.getFromBucket(); final String toBucket = this.step.getToBucket(); final String fromPath = this.step.getFromPath(); final String toPath = this.step.getToPath(); final String kmsId = this.step.getKmsId(); final Map<String, String> metadatas = new HashMap<>(); final CannedAccessControlList acl = this.step.getAcl(); final String cacheControl = this.step.getCacheControl(); final String contentType = this.step.getContentType(); final String sseAlgorithm = this.step.getSseAlgorithm(); final S3ClientOptions s3ClientOptions = this.step.createS3ClientOptions(); final EnvVars envVars = this.getContext().get(EnvVars.class); if (this.step.getMetadatas() != null && this.step.getMetadatas().length != 0) { for (String metadata : this.step.getMetadatas()) { if (metadata.split(":").length == 2) { metadatas.put(metadata.split(":")[0], metadata.split(":")[1]); } } } Preconditions.checkArgument(fromBucket != null && !fromBucket.isEmpty(), "From bucket must not be null or empty"); Preconditions.checkArgument(fromPath != null && !fromPath.isEmpty(), "From path must not be null or empty"); Preconditions.checkArgument(toBucket != null && !toBucket.isEmpty(), "To bucket must not be null or empty"); Preconditions.checkArgument(toPath != null && !toPath.isEmpty(), "To path must not be null or empty"); TaskListener listener = Execution.this.getContext().get(TaskListener.class); listener.getLogger().format("Copying s3://%s/%s to s3://%s/%s%n", fromBucket, fromPath, toBucket, toPath); CopyObjectRequest request = new CopyObjectRequest(fromBucket, fromPath, toBucket, toPath); // Add metadata if (metadatas.size() > 0 || (cacheControl != null && !cacheControl.isEmpty()) || (contentType != null && !contentType.isEmpty()) || (sseAlgorithm != null && !sseAlgorithm.isEmpty())) { ObjectMetadata metas = new ObjectMetadata(); if (metadatas.size() > 0) { metas.setUserMetadata(metadatas); } if (cacheControl != null && !cacheControl.isEmpty()) { metas.setCacheControl(cacheControl); } if (contentType != null && !contentType.isEmpty()) { metas.setContentType(contentType); } if (sseAlgorithm != null && !sseAlgorithm.isEmpty()) { metas.setSSEAlgorithm(sseAlgorithm); } request.withNewObjectMetadata(metas); } // Add acl if (acl != null) { request.withCannedAccessControlList(acl); } // Add kms if (kmsId != null && !kmsId.isEmpty()) { listener.getLogger().format("Using KMS: %s%n", kmsId); request.withSSEAwsKeyManagementParams(new SSEAwsKeyManagementParams(kmsId)); } TransferManager mgr = TransferManagerBuilder.standard() .withS3Client(AWSClientFactory.create(s3ClientOptions.createAmazonS3ClientBuilder(), envVars)) .build(); try { final Copy copy = mgr.copy(request); copy.addProgressListener((ProgressListener) progressEvent -> { if (progressEvent.getEventType() == ProgressEventType.TRANSFER_COMPLETED_EVENT) { listener.getLogger().println("Finished: " + copy.getDescription()); } }); copy.waitForCompletion(); } finally{ mgr.shutdownNow(); } listener.getLogger().println("Copy complete"); return String.format("s3://%s/%s", toBucket, toPath); }
Example #26
Source File: S3Module.java From cassandra-backup with Apache License 2.0 | 4 votes |
public TransferManager build(final AbstractOperationRequest operationRequest) { final AmazonS3 amazonS3 = provideAmazonS3(coreV1ApiProvider, operationRequest); return TransferManagerBuilder.standard().withS3Client(amazonS3).build(); }
Example #27
Source File: S3Accessor.java From datacollector with Apache License 2.0 | 4 votes |
TransferManagerBuilder createTransferManagerBuilder() { return TransferManagerBuilder.standard(); }
Example #28
Source File: AmazonS3Target.java From datacollector with Apache License 2.0 | 4 votes |
@Override public List<ConfigIssue> init() { List<ConfigIssue> issues = s3TargetConfigBean.init(getContext(), super.init(), isErrorStage); elVars = getContext().createELVars(); bucketEval = getContext().createELEval(BUCKET_TEMPLATE); partitionEval = getContext().createELEval(PARTITION_TEMPLATE); timeDriverEval = getContext().createELEval(TIME_DRIVER_TEMPLATE); calendar = Calendar.getInstance(TimeZone.getTimeZone(ZoneId.of(s3TargetConfigBean.timeZoneID))); transferManager = TransferManagerBuilder .standard() .withS3Client(s3TargetConfigBean.s3Config.getS3Client()) .withExecutorFactory(() -> Executors.newFixedThreadPool(s3TargetConfigBean.tmConfig.threadPoolSize)) .withMinimumUploadPartSize(s3TargetConfigBean.tmConfig.minimumUploadPartSize) .withMultipartUploadThreshold(s3TargetConfigBean.tmConfig.multipartUploadThreshold) .build(); TimeEL.setCalendarInContext(elVars, calendar); TimeNowEL.setTimeNowInContext(elVars, new Date()); if (partitionTemplate.contains(EL_PREFIX)) { ELUtils.validateExpression(partitionTemplate, getContext(), Groups.S3.getLabel(), S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + PARTITION_TEMPLATE, Errors.S3_03, issues ); } if (timeDriverTemplate.contains(EL_PREFIX)) { ELUtils.validateExpression(timeDriverTemplate, getContext(), Groups.S3.getLabel(), S3TargetConfigBean.S3_TARGET_CONFIG_BEAN_PREFIX + TIME_DRIVER_TEMPLATE, Errors.S3_04, issues ); } if (!isErrorStage && getContext().getService(DataFormatGeneratorService.class).isWholeFileFormat()) { fileHelper = new WholeFileHelper(getContext(), s3TargetConfigBean, transferManager, issues); } else { fileHelper = new DefaultFileHelper(getContext(), s3TargetConfigBean, transferManager, isErrorStage); } this.errorRecordHandler = new DefaultErrorRecordHandler(getContext()); return issues; }
Example #29
Source File: AwsS3Storage.java From ecs-sync with Apache License 2.0 | 4 votes |
@Override void putObject(SyncObject obj, String targetKey) { ObjectMetadata om; if (options.isSyncMetadata()) om = s3MetaFromSyncMeta(obj.getMetadata()); else om = new ObjectMetadata(); if (obj.getMetadata().isDirectory()) om.setContentType(TYPE_DIRECTORY); PutObjectRequest req; File file = (File) obj.getProperty(AbstractFilesystemStorage.PROP_FILE); S3ProgressListener progressListener = null; if (obj.getMetadata().isDirectory()) { req = new PutObjectRequest(config.getBucketName(), targetKey, new ByteArrayInputStream(new byte[0]), om); } else if (file != null) { req = new PutObjectRequest(config.getBucketName(), targetKey, file).withMetadata(om); progressListener = new ByteTransferListener(obj); } else { InputStream stream = obj.getDataStream(); if (options.isMonitorPerformance()) stream = new ProgressInputStream(stream, new PerformanceListener(getWriteWindow())); req = new PutObjectRequest(config.getBucketName(), targetKey, stream, om); } if (options.isSyncAcl()) req.setAccessControlList(s3AclFromSyncAcl(obj.getAcl(), options.isIgnoreInvalidAcls())); TransferManager xferManager = null; try { // xfer manager will figure out if MPU is needed (based on threshold), do the MPU if necessary, // and abort if it fails xferManager = TransferManagerBuilder.standard() .withS3Client(s3) .withExecutorFactory(() -> Executors.newFixedThreadPool(config.getMpuThreadCount())) .withMultipartUploadThreshold((long) config.getMpuThresholdMb() * 1024 * 1024) .withMinimumUploadPartSize((long) config.getMpuPartSizeMb() * 1024 * 1024) .withShutDownThreadPools(true) .build(); // directly update final Upload upload = xferManager.upload(req, progressListener); try { String eTag = time((Callable<String>) () -> upload.waitForUploadResult().getETag(), OPERATION_MPU); log.debug("Wrote {}, etag: {}", targetKey, eTag); } catch (Exception e) { log.error("upload exception", e); if (e instanceof RuntimeException) throw (RuntimeException) e; throw new RuntimeException("upload thread was interrupted", e); } } finally { // NOTE: apparently if we do not reference xferManager again after the upload() call (as in this finally // block), the JVM will for some crazy reason determine it is eligible for GC and call finalize(), which // shuts down the thread pool, fails the upload, and gives absolutely no indication of what's going on... if (xferManager != null) xferManager.shutdownNow(false); } }
Example #30
Source File: S3SchemaUriResolver.java From data-highway with Apache License 2.0 | 4 votes |
public S3SchemaUriResolver(AmazonS3 s3Client, String bucket, String keyPrefix, boolean enableServerSideEncryption) { this(TransferManagerBuilder.standard().withS3Client(s3Client).build(), S3_URI_FORMAT, bucket, keyPrefix, enableServerSideEncryption); }