com.amazonaws.services.s3.AmazonS3 Java Examples

The following examples show how to use com.amazonaws.services.s3.AmazonS3. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3Source.java    From sequenceiq-samples with Apache License 2.0 7 votes vote down vote up
@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 
Example #2
Source File: S3Configuration.java    From data-highway with Apache License 2.0 6 votes vote down vote up
@Bean
AbortableOutputStreamFactory s3OutputStreamFactory(
    AmazonS3 s3,
    @Value("${s3.bucket}") String bucket,
    @Value("${s3.partSize:5242880}") int partSize, // default 5 mebibytes
    @Value("${s3.enableServerSideEncryption:false}") boolean enableServerSideEncryption,
    @Value("${s3.retry.maxAttempts:3}") int maxAttempts,
    @Value("${s3.retry.sleepSeconds:1}") int sleepSeconds,
    @Value("${s3.async.poolSize:3}") int poolSize,
    @Value("${s3.async.queueSize:3}") int queueSize) {
  return key -> S3MultipartOutputStream
      .builder()
      .s3(s3, bucket, key)
      .partSize(partSize)
      .enableServerSideEncryption(enableServerSideEncryption)
      .retries(maxAttempts, sleepSeconds)
      .async(poolSize, queueSize)
      .build();
}
 
Example #3
Source File: S3Encrypt.java    From aws-doc-sdk-examples with Apache License 2.0 6 votes vote down vote up
/**
 * Uses AES/GCM with AESWrap key wrapping to encrypt the key. Uses v2 metadata schema. The only difference between this and
 * {@link #authenticatedEncryption_CustomerManagedKey()} is that attempting to retrieve an object non
 * encrypted with AES/GCM will thrown an exception instead of falling back to encryption only or plaintext GET.
 */
// snippet-start:[s3.java1.s3_encrypt.strict_authenticated_encryption]
public void strictAuthenticatedEncryption_CustomerManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.strict_authenticated_encryption_build]
    SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey();
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.StrictAuthenticatedEncryption))
            .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey)))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.strict_authenticated_encryption_build]

    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    try {
        s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY);
    } catch (SecurityException e) {
        // Strict authenticated encryption will throw an exception if an object is not encrypted with AES/GCM
        System.err.println(NON_ENCRYPTED_KEY + " was not encrypted with AES/GCM");
    }
}
 
Example #4
Source File: S3BlockSpiller.java    From aws-athena-query-federation with Apache License 2.0 6 votes vote down vote up
/**
 * Constructs a new S3BlockSpiller.
 *
 * @param amazonS3 AmazonS3 client to use for writing to S3.
 * @param spillConfig The spill config for this instance. Includes things like encryption key, s3 path, etc...
 * @param allocator The BlockAllocator to use when creating blocks.
 * @param schema The schema for blocks that should be written.
 * @param constraintEvaluator The ConstraintEvaluator that should be used to constrain writes.
 * @param maxRowsPerCall The max number of rows to allow callers to write in one call.
 */
public S3BlockSpiller(AmazonS3 amazonS3,
        SpillConfig spillConfig,
        BlockAllocator allocator,
        Schema schema,
        ConstraintEvaluator constraintEvaluator,
        int maxRowsPerCall)
{
    this.amazonS3 = requireNonNull(amazonS3, "amazonS3 was null");
    this.spillConfig = requireNonNull(spillConfig, "spillConfig was null");
    this.allocator = requireNonNull(allocator, "allocator was null");
    this.schema = requireNonNull(schema, "schema was null");
    this.blockCrypto = (spillConfig.getEncryptionKey() != null) ? new AesGcmBlockCrypto(allocator) : new NoOpBlockCrypto(allocator);
    asyncSpillPool = (spillConfig.getNumSpillThreads() <= 0) ? null :
            Executors.newFixedThreadPool(spillConfig.getNumSpillThreads());
    this.maxRowsPerCall = maxRowsPerCall;
    this.constraintEvaluator = constraintEvaluator;
}
 
Example #5
Source File: AwsS3BucketService.java    From pacbot with Apache License 2.0 6 votes vote down vote up
public boolean uploadFile(final AmazonS3 amazonS3, MultipartFile fileToUpload, String s3BucketName, String key) {
	try {
		File file = AdminUtils.convert(fileToUpload);
		long size = fileToUpload.getSize();
		String contentType = fileToUpload.getContentType();
		ObjectMetadata metadata = new ObjectMetadata();
		metadata.setContentType(contentType);
		metadata.setContentLength(size);
		PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, key, file).withCannedAcl(CannedAccessControlList.PublicRead);
		amazonS3.putObject(putObjectRequest);
		return Boolean.TRUE;
	} catch (IOException exception) {
		log.error(UNEXPECTED_ERROR_OCCURRED, exception);
	} 
	return Boolean.FALSE;
}
 
Example #6
Source File: S3ReadableSeekableByteChannel.java    From beam with Apache License 2.0 6 votes vote down vote up
S3ReadableSeekableByteChannel(AmazonS3 amazonS3, S3ResourceId path, S3Options options)
    throws IOException {
  this.amazonS3 = checkNotNull(amazonS3, "amazonS3");
  checkNotNull(path, "path");
  this.options = checkNotNull(options, "options");

  if (path.getSize().isPresent()) {
    contentLength = path.getSize().get();
    this.path = path;

  } else {
    try {
      contentLength =
          amazonS3.getObjectMetadata(path.getBucket(), path.getKey()).getContentLength();
    } catch (AmazonClientException e) {
      throw new IOException(e);
    }
    this.path = path.withSize(contentLength);
  }
}
 
Example #7
Source File: S3Connection.java    From components with Apache License 2.0 6 votes vote down vote up
private static String getEndpoint(AmazonS3 s3client, String bucket) {
    String bucketLocation = null;
    try { 
        bucketLocation = s3client.getBucketLocation(bucket);
    } catch(IllegalArgumentException e) {
        //java.lang.IllegalArgumentException: Cannot create enum from eu-west-2 value!
        String info = e.getMessage();
        if(info == null || info.isEmpty()) {
            throw e;
        }
        Pattern regex = Pattern.compile("[a-zA-Z]+-[a-zA-Z]+-[1-9]");
        Matcher matcher = regex.matcher(info);
        if(matcher.find()) {
            bucketLocation = matcher.group(0);
        } else {
            throw e;
        }
    }
    String region = S3RegionUtil.getBucketRegionFromLocation(bucketLocation);
    return S3RegionUtil.regionToEndpoint(region);
}
 
Example #8
Source File: StashSplitIterator.java    From emodb with Apache License 2.0 6 votes vote down vote up
StashSplitIterator(AmazonS3 s3, String bucket, String key) {
    InputStream rawIn = new RestartingS3InputStream(s3, bucket, key);
    try {
        // File is gzipped
        // Note:
        //   Because the content may be concatenated gzip files we cannot use the default GZIPInputStream.
        //   GzipCompressorInputStream supports concatenated gzip files.
        GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(rawIn, true);
        _in = new BufferedReader(new InputStreamReader(gzipIn, Charsets.UTF_8));
        // Create a line reader
        _reader = new LineReader(_in);
    } catch (Exception e) {
        try {
            Closeables.close(rawIn, true);
        } catch (IOException ignore) {
            // Won't happen, already caught and logged
        }
        throw Throwables.propagate(e);
    }
}
 
Example #9
Source File: S3ChangeLogStore.java    From athenz with Apache License 2.0 6 votes vote down vote up
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("getSignedDomain with S3: {}", domainName);
        }
        
        SignedDomain signedDomain = null;
        try {
            S3Object object = s3.getObject(s3BucketName, domainName);
            try (S3ObjectInputStream s3is = object.getObjectContent()) {
                signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
            }
        } catch (Exception ex) {
            LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain {} error: {}",
                    domainName, ex.getMessage());
        }
        return signedDomain;
    }
 
Example #10
Source File: S3Util.java    From teamcity-s3-artifact-storage-plugin with Apache License 2.0 6 votes vote down vote up
public static <T> T withClientCorrectingRegion(@NotNull final AmazonS3 s3Client,
                                               @NotNull final Map<String, String> settings,
                                               @NotNull final WithS3<T, AmazonS3Exception> withCorrectedClient) {
  try {
    return withCorrectedClient.run(s3Client);
  } catch (AmazonS3Exception awsException) {
    final String correctRegion = extractCorrectedRegion(awsException);
    if (correctRegion != null) {
      LOGGER.debug("Running operation with corrected S3 region [" + correctRegion + "]", awsException);
      final HashMap<String, String> correctedSettings = new HashMap<>(settings);
      correctedSettings.put(REGION_NAME_PARAM, correctRegion);
      return withS3Client(correctedSettings, withCorrectedClient);
    } else {
      throw awsException;
    }
  }
}
 
Example #11
Source File: AmazonS3OAuthStateService.java    From java-slack-sdk with MIT License 6 votes vote down vote up
@Override
public boolean isAvailableInDatabase(String state) {
    AmazonS3 s3 = this.createS3Client();
    S3Object s3Object = getObject(s3, getKey(state));
    if (s3Object == null) {
        return false;
    }
    String millisToExpire = null;
    try {
        millisToExpire = IOUtils.toString(s3Object.getObjectContent());
        return Long.valueOf(millisToExpire) > System.currentTimeMillis();
    } catch (IOException e) {
        log.error("Failed to load a state data for state: {}", state, e);
        return false;
    } catch (NumberFormatException ne) {
        log.error("Invalid state value detected - state: {}, millisToExpire: {}", state, millisToExpire);
        return false;
    }
}
 
Example #12
Source File: PrimitiveS3OperationHandler.java    From CloverETL-Engine with GNU Lesser General Public License v2.1 6 votes vote down vote up
/**
 * Deletes a regular file.
 */
@Override
public boolean deleteFile(URI target) throws IOException {
	target = target.normalize();
	PooledS3Connection connection = null;
	try {
		connection = connect(target);
		AmazonS3 service = connection.getService();
		String[] path = getPath(target);
		try {
			service.deleteObject(path[0], path[1]);
			return true;
		} catch (AmazonClientException e) {
			throw new IOException(e);
		}
	} finally {
		disconnect(connection);
	}
}
 
Example #13
Source File: S3.java    From rdf-delta with Apache License 2.0 6 votes vote down vote up
public static AmazonS3 buildS3(LocalServerConfig configuration) {
    String region = configuration.getProperty(pRegion);
    String endpoint = configuration.getProperty(pEndpoint);
    String credentialsFile =  configuration.getProperty(pCredentialFile);
    String credentialsProfile =  configuration.getProperty(pCredentialProfile);

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    if ( endpoint == null )
        builder.withRegion(region);
    else  {
        // Needed for S3mock
        builder.withPathStyleAccessEnabled(true);
        builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region));
        builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
    }
    if ( credentialsFile != null )
        builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile));
    return builder.build();
}
 
Example #14
Source File: S3Validator.java    From halyard with Apache License 2.0 6 votes vote down vote up
@Override
public void validate(ConfigProblemSetBuilder ps, S3PersistentStore n) {
  if (!StringUtils.isEmpty(n.getEndpoint())) {
    return;
  }

  try {
    AWSCredentialsProvider credentialsProvider =
        AwsAccountValidator.getAwsCredentialsProvider(
            n.getAccessKeyId(), secretSessionManager.decrypt(n.getSecretAccessKey()));
    S3Config s3Config = new S3Config();
    S3MetadataStorageProperties s3Properties = new S3MetadataStorageProperties();
    s3Properties.setBucket(n.getBucket());
    s3Properties.setRootFolder(n.getRootFolder());
    s3Properties.setRegion(n.getRegion());
    AmazonS3 s3Client = s3Config.awsS3MetadataClient(credentialsProvider, s3Properties);
    new S3Config().s3StorageService(s3Client, s3Properties);
  } catch (Exception e) {
    ps.addProblem(
        Problem.Severity.ERROR,
        "Failed to ensure the required bucket \""
            + n.getBucket()
            + "\" exists: "
            + e.getMessage());
  }
}
 
Example #15
Source File: AmazonS3InstallationServiceTest.java    From java-slack-sdk with MIT License 6 votes vote down vote up
@Test
public void initializer() {
    AWSCredentials credentials = mock(AWSCredentials.class);
    when(credentials.getAWSAccessKeyId()).thenReturn("valid key");
    AmazonS3 s3 = mock(AmazonS3.class);
    when(s3.doesBucketExistV2(anyString())).thenReturn(true);

    AmazonS3InstallationService service = new AmazonS3InstallationService("test-bucket") {
        @Override
        protected AWSCredentials getCredentials() {
            return credentials;
        }

        @Override
        protected AmazonS3 createS3Client() {
            return s3;
        }
    };
    service.initializer().accept(null);
}
 
Example #16
Source File: AmazonS3SourceConfiguration.java    From spring-cloud-stream-app-starters with Apache License 2.0 6 votes vote down vote up
@Bean
public S3InboundFileSynchronizer s3InboundFileSynchronizer(AmazonS3 amazonS3,
		ResourceIdResolver resourceIdResolver) {
	S3SessionFactory s3SessionFactory = new S3SessionFactory(amazonS3, resourceIdResolver);
	S3InboundFileSynchronizer synchronizer = new S3InboundFileSynchronizer(s3SessionFactory);
	synchronizer.setDeleteRemoteFiles(this.s3SourceProperties.isDeleteRemoteFiles());
	synchronizer.setPreserveTimestamp(this.s3SourceProperties.isPreserveTimestamp());
	String remoteDir = this.s3SourceProperties.getRemoteDir();
	synchronizer.setRemoteDirectory(remoteDir);
	synchronizer.setRemoteFileSeparator(this.s3SourceProperties.getRemoteFileSeparator());
	synchronizer.setTemporaryFileSuffix(this.s3SourceProperties.getTmpFileSuffix());

	if (StringUtils.hasText(this.s3SourceProperties.getFilenamePattern())) {
		synchronizer.setFilter(new S3SimplePatternFileListFilter(this.s3SourceProperties.getFilenamePattern()));
	}
	else if (this.s3SourceProperties.getFilenameRegex() != null) {
		synchronizer.setFilter(new S3RegexPatternFileListFilter(this.s3SourceProperties.getFilenameRegex()));
	}

	return synchronizer;
}
 
Example #17
Source File: S3BlobStorage.java    From mojito with Apache License 2.0 5 votes vote down vote up
public S3BlobStorage(AmazonS3 amazonS3,
                     S3BlobStorageConfigurationProperties s3BlobStorageConfigurationProperties) {
    Preconditions.checkNotNull(amazonS3);
    Preconditions.checkNotNull(s3BlobStorageConfigurationProperties);

    this.amazonS3 = amazonS3;
    this.s3BlobStorageConfigurationProperties = s3BlobStorageConfigurationProperties;
}
 
Example #18
Source File: S3ClientFactory.java    From front50 with Apache License 2.0 5 votes vote down vote up
public static AmazonS3 create(
    AWSCredentialsProvider awsCredentialsProvider, S3Properties s3Properties) {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  if (s3Properties.getProxyProtocol() != null) {
    if (s3Properties.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
      clientConfiguration.setProtocol(Protocol.HTTPS);
    } else {
      clientConfiguration.setProtocol(Protocol.HTTP);
    }
    Optional.ofNullable(s3Properties.getProxyHost()).ifPresent(clientConfiguration::setProxyHost);
    Optional.ofNullable(s3Properties.getProxyPort())
        .map(Integer::parseInt)
        .ifPresent(clientConfiguration::setProxyPort);
  }

  AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);

  if (!StringUtils.isEmpty(s3Properties.getEndpoint())) {
    client.setEndpoint(s3Properties.getEndpoint());

    if (!StringUtils.isEmpty(s3Properties.getRegionOverride())) {
      client.setSignerRegionOverride(s3Properties.getRegionOverride());
    }

    client.setS3ClientOptions(
        S3ClientOptions.builder().setPathStyleAccess(s3Properties.getPathStyleAccess()).build());
  } else {
    Optional.ofNullable(s3Properties.getRegion())
        .map(Regions::fromName)
        .map(Region::getRegion)
        .ifPresent(client::setRegion);
  }

  return client;
}
 
Example #19
Source File: TruckParkAppIntegrationTest.java    From data-highway with Apache License 2.0 5 votes vote down vote up
@Primary
@Bean
AmazonS3 testS3(@Value("${s3.port}") int port) {
  return AmazonS3Client
      .builder()
      .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
      .withEndpointConfiguration(new EndpointConfiguration("http://127.0.0.1:" + port, "us-west-2"))
      .build();
}
 
Example #20
Source File: S3UploadStepTransferManagerIntegrationTest.java    From pipeline-aws-plugin with Apache License 2.0 5 votes vote down vote up
@Before
public void setupSdk() throws Exception {
	PowerMockito.mockStatic(AWSClientFactory.class);
	AmazonS3 amazonS3 = PowerMockito.mock(AmazonS3.class);
	PowerMockito.when(AWSClientFactory.create(Mockito.any(AwsSyncClientBuilder.class), Mockito.any(StepContext.class)))
			.thenReturn(amazonS3);

	PowerMockito.mockStatic(TransferManagerBuilder.class);
	transferManager = Mockito.mock(TransferManager.class);
	TransferManagerBuilder transferManagerBuilder = PowerMockito.mock(TransferManagerBuilder.class);
	PowerMockito.when(TransferManagerBuilder.standard()).thenReturn(transferManagerBuilder);
	PowerMockito.when(transferManagerBuilder.withS3Client(Mockito.any(AmazonS3.class))).thenReturn(transferManagerBuilder);
	PowerMockito.when(transferManagerBuilder.build()).thenReturn(transferManager);
}
 
Example #21
Source File: SimpleStorageRangeResource.java    From genie with Apache License 2.0 5 votes vote down vote up
SimpleStorageRangeResource(
    final AmazonS3 client,
    final String bucket,
    final String key,
    final String versionId, final TaskExecutor s3TaskExecutor,
    final Pair<Integer, Integer> range
) throws IOException {
    super(client, bucket, key, s3TaskExecutor, versionId);
    this.client =  AmazonS3ProxyFactory.createProxy(client);
    this.bucket = bucket;
    this.key = key;
    this.versionId = versionId;
    this.range = range;

    long tempContentLength = -1;
    try {
        tempContentLength = super.contentLength();
    } catch (FileNotFoundException e) {
        // S3 object does not exist.
        // Upstream code will handle this correctly by checking exists(), contentLength(), etc.
        log.warn("Returning non-existent S3 resource {}/{}", bucket, key);
    }
    this.contentLength = tempContentLength;

    final Integer lower = this.range.getLeft();
    final Integer upper = this.range.getRight();

    if ((lower != null && upper != null && lower > upper)
        || (lower != null && lower > contentLength)) {
        throw new IllegalArgumentException(
            "Invalid range " + lower + "-" + upper + " for S3 object of size " + contentLength
        );
    }
}
 
Example #22
Source File: InventoryReportLineWriter.java    From s3-inventory-usage-examples with Apache License 2.0 5 votes vote down vote up
public InventoryReportLineWriter(AmazonS3 client, String destBucketName, String destPrefix,
                                 String srcBucket, InventoryManifest inventoryManifest) throws IOException{
    this.s3Client = client;
    this.bucketName = destBucketName;
    String uuid = UUID.randomUUID().toString();
    this.outputInventoryReportKey = destPrefix + "/" + srcBucket + "/data/" + uuid + ".csv.gz";
    this.schema = CsvSchemaFactory.buildSchema(inventoryManifest);
}
 
Example #23
Source File: S3StorageService.java    From kayenta with Apache License 2.0 5 votes vote down vote up
/** Check to see if the bucket exists, creating it if it is not there. */
public void ensureBucketExists(String accountName) {
  AwsNamedAccountCredentials credentials =
      accountCredentialsRepository.getRequiredOne(accountName);

  AmazonS3 amazonS3 = credentials.getAmazonS3();
  String bucket = credentials.getBucket();
  String region = credentials.getRegion();

  HeadBucketRequest request = new HeadBucketRequest(bucket);

  try {
    amazonS3.headBucket(request);
  } catch (AmazonServiceException e) {
    if (e.getStatusCode() == 404) {
      if (com.amazonaws.util.StringUtils.isNullOrEmpty(region)) {
        log.warn("Bucket {} does not exist. Creating it in default region.", bucket);
        amazonS3.createBucket(bucket);
      } else {
        log.warn("Bucket {} does not exist. Creating it in region {}.", bucket, region);
        amazonS3.createBucket(bucket, region);
      }
    } else {
      log.error("Could not create bucket {}: {}", bucket, e);
      throw e;
    }
  }
}
 
Example #24
Source File: AWSClientUtils.java    From camel-kafka-connector with Apache License 2.0 5 votes vote down vote up
public static AmazonS3 newS3Client() {
    LOG.debug("Creating a new S3 client");
    AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();

    String awsInstanceType = System.getProperty("aws-service.instance.type");
    String region = getRegion();

    if (awsInstanceType == null || awsInstanceType.equals("local-aws-container")) {
        String amazonHost = System.getProperty(AWSConfigs.AMAZON_AWS_HOST);
        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setProtocol(Protocol.HTTP);

        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(amazonHost, region))
                .withClientConfiguration(clientConfiguration)
                .withCredentials(new TestAWSCredentialsProvider("accesskey", "secretkey"));
    } else {
        clientBuilder
                .withRegion(region)
                .withCredentials(new TestAWSCredentialsProvider());
    }

    clientBuilder
            .withPathStyleAccessEnabled(true);

    return clientBuilder.build();
}
 
Example #25
Source File: Main.java    From java-almanac with Creative Commons Attribution Share Alike 4.0 International 5 votes vote down vote up
private static IMultiReportOutput createS3Output() {
	AmazonS3 s3Client = AmazonS3ClientBuilder.standard() //
			.withRegion("us-east-1") //
			.withCredentials(new SystemPropertiesCredentialsProvider()) //
			.build();
	return new S3MultiReportOutput(s3Client, "download.eclipselab.org", "jdkdiff/");
}
 
Example #26
Source File: MCAWS.java    From aws-big-data-blog with Apache License 2.0 5 votes vote down vote up
public static void putImageS3(String bucketName, String key, String fileName) {
    AmazonS3 s3 = new AmazonS3Client();
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    //Region usWest2 = Region.getRegion(s3Region);
    s3.setRegion(usWest2);
    try {
        File file = new File(fileName);
        s3.putObject(new PutObjectRequest(bucketName, key, file));
    } catch (Exception e) { System.out.println("ERROR ON IMAGE FILE"); }
}
 
Example #27
Source File: S3OptimizedFileInputFormat.java    From kangaroo with Apache License 2.0 5 votes vote down vote up
@Override
protected List<FileStatus> listStatus(final JobContext job) throws IOException {
    final Path[] dirs = getInputPaths(job);
    if (dirs.length == 0) {
        throw new IOException("No input paths specified in job");
    }
    final long blockSize = job.getConfiguration().getLong(S3NativeFileSystemConfigKeys.S3_NATIVE_BLOCK_SIZE_KEY,
            S3NativeFileSystemConfigKeys.S3_NATIVE_BLOCK_SIZE_DEFAULT);
    final AmazonS3 s3Client = S3HadoopUtils.getS3Client(job.getConfiguration());
    return S3InputFormatUtils.getFileStatuses(s3Client, blockSize, dirs);
}
 
Example #28
Source File: S3CleanupExtension.java    From teamcity-s3-artifact-storage-plugin with Apache License 2.0 5 votes vote down vote up
@NotNull
private Integer deleteChunk(@NotNull final String pathPrefix,
                            @NotNull final String bucketName,
                            @NotNull final AmazonS3 client,
                            @NotNull final List<String> part,
                            @NotNull final Supplier<String> info) throws Exception {
  final List<DeleteObjectsRequest.KeyVersion> objectKeys = part.stream().map(path -> new DeleteObjectsRequest.KeyVersion(pathPrefix + path)).collect(Collectors.toList());
  final DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName).withKeys(objectKeys);
  return executeWithNewThreadName(info.get(),
                                  () -> doUnderContextClassLoader(S3Util.class.getClassLoader(), () -> client.deleteObjects(deleteObjectsRequest).getDeletedObjects().size()));
}
 
Example #29
Source File: MultipartUploader.java    From nexus-public with Eclipse Public License 1.0 5 votes vote down vote up
@Override
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
  try (InputStream input = contents) {
    InputStream chunkOne = readChunk(input);
    if (chunkOne.available() < chunkSize) {
      uploadSinglePart(s3, bucket, key, chunkOne);
    }
    else {
      uploadMultiPart(s3, bucket, key, chunkOne, contents);
    }
  }
  catch(IOException | SdkClientException e) { // NOSONAR
    throw new BlobStoreException("Error uploading blob", e, null);
  }
}
 
Example #30
Source File: AmazonS3Config.java    From ReCiter with Apache License 2.0 5 votes vote down vote up
/**
 * @return AmazonS3 client object
 */
@Bean
@Scope("singleton")
public AmazonS3 amazonS3() {
	
	if(isS3Use && !isDynamoDbLocal) {
 	final AmazonS3 s3 = AmazonS3ClientBuilder
 						.standard()
 						.withCredentials(new AWSStaticCredentialsProvider(new AWSCredentials() {
					
					@Override
					public String getAWSSecretKey() {
						return amazonAWSSecretKey;
					}
					
					@Override
					public String getAWSAccessKeyId() {
						return amazonAWSAccessKey;
					}
				}))
 						.withRegion(awsS3Region)
 						.build();
 	createBucket(s3);
 	return s3;
	}
						
	return null;		
}