Java Code Examples for com.amazonaws.services.s3.AmazonS3

The following examples show how to use com.amazonaws.services.s3.AmazonS3. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: sequenceiq-samples   Source File: S3Source.java    License: Apache License 2.0 7 votes vote down vote up
@Override
protected void doStart() {
    AWSCredentials myCredentials = new BasicAWSCredentials(accessKey, secretKey);
    AmazonS3 s3Client = new AmazonS3Client(myCredentials);
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);
    ChannelProcessor channelProcessor = getChannelProcessor();
    for (S3ObjectSummary s3ObjectSummary : objectListing.getObjectSummaries()) {
        String file = s3ObjectSummary.getKey();
        LOGGER.info("Read the content of {}", file);
        GetObjectRequest objectRequest = new GetObjectRequest(bucket, file);
        S3Object objectPortion = s3Client.getObject(objectRequest);
        try {
            long startTime = System.currentTimeMillis();
            processLines(channelProcessor, objectPortion.getObjectContent());
            LOGGER.info("Processing of {} took {} ms", file, System.currentTimeMillis() - startTime);
        } catch (IOException e) {
            LOGGER.warn("Cannot process the {}, skipping", file, e);
        }
    }
}
 
Example 2
Source Project: data-highway   Source File: S3Configuration.java    License: Apache License 2.0 6 votes vote down vote up
@Bean
AbortableOutputStreamFactory s3OutputStreamFactory(
    AmazonS3 s3,
    @Value("${s3.bucket}") String bucket,
    @Value("${s3.partSize:5242880}") int partSize, // default 5 mebibytes
    @Value("${s3.enableServerSideEncryption:false}") boolean enableServerSideEncryption,
    @Value("${s3.retry.maxAttempts:3}") int maxAttempts,
    @Value("${s3.retry.sleepSeconds:1}") int sleepSeconds,
    @Value("${s3.async.poolSize:3}") int poolSize,
    @Value("${s3.async.queueSize:3}") int queueSize) {
  return key -> S3MultipartOutputStream
      .builder()
      .s3(s3, bucket, key)
      .partSize(partSize)
      .enableServerSideEncryption(enableServerSideEncryption)
      .retries(maxAttempts, sleepSeconds)
      .async(poolSize, queueSize)
      .build();
}
 
Example 3
Source Project: aws-doc-sdk-examples   Source File: S3Encrypt.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Uses AES/GCM with AESWrap key wrapping to encrypt the key. Uses v2 metadata schema. The only difference between this and
 * {@link #authenticatedEncryption_CustomerManagedKey()} is that attempting to retrieve an object non
 * encrypted with AES/GCM will thrown an exception instead of falling back to encryption only or plaintext GET.
 */
// snippet-start:[s3.java1.s3_encrypt.strict_authenticated_encryption]
public void strictAuthenticatedEncryption_CustomerManagedKey() throws NoSuchAlgorithmException {
    // snippet-start:[s3.java1.s3_encrypt.strict_authenticated_encryption_build]
    SecretKey secretKey = KeyGenerator.getInstance("AES").generateKey();
    AmazonS3Encryption s3Encryption = AmazonS3EncryptionClientBuilder
            .standard()
            .withRegion(Regions.US_WEST_2)
            .withCryptoConfiguration(new CryptoConfiguration(CryptoMode.StrictAuthenticatedEncryption))
            .withEncryptionMaterials(new StaticEncryptionMaterialsProvider(new EncryptionMaterials(secretKey)))
            .build();

    AmazonS3 s3NonEncrypt = AmazonS3ClientBuilder.standard().withRegion(Regions.DEFAULT_REGION).build();
    // snippet-end:[s3.java1.s3_encrypt.strict_authenticated_encryption_build]

    s3Encryption.putObject(BUCKET_NAME, ENCRYPTED_KEY, "some contents");
    s3NonEncrypt.putObject(BUCKET_NAME, NON_ENCRYPTED_KEY, "some other contents");
    System.out.println(s3Encryption.getObjectAsString(BUCKET_NAME, ENCRYPTED_KEY));
    try {
        s3Encryption.getObjectAsString(BUCKET_NAME, NON_ENCRYPTED_KEY);
    } catch (SecurityException e) {
        // Strict authenticated encryption will throw an exception if an object is not encrypted with AES/GCM
        System.err.println(NON_ENCRYPTED_KEY + " was not encrypted with AES/GCM");
    }
}
 
Example 4
/**
 * Constructs a new S3BlockSpiller.
 *
 * @param amazonS3 AmazonS3 client to use for writing to S3.
 * @param spillConfig The spill config for this instance. Includes things like encryption key, s3 path, etc...
 * @param allocator The BlockAllocator to use when creating blocks.
 * @param schema The schema for blocks that should be written.
 * @param constraintEvaluator The ConstraintEvaluator that should be used to constrain writes.
 * @param maxRowsPerCall The max number of rows to allow callers to write in one call.
 */
public S3BlockSpiller(AmazonS3 amazonS3,
        SpillConfig spillConfig,
        BlockAllocator allocator,
        Schema schema,
        ConstraintEvaluator constraintEvaluator,
        int maxRowsPerCall)
{
    this.amazonS3 = requireNonNull(amazonS3, "amazonS3 was null");
    this.spillConfig = requireNonNull(spillConfig, "spillConfig was null");
    this.allocator = requireNonNull(allocator, "allocator was null");
    this.schema = requireNonNull(schema, "schema was null");
    this.blockCrypto = (spillConfig.getEncryptionKey() != null) ? new AesGcmBlockCrypto(allocator) : new NoOpBlockCrypto(allocator);
    asyncSpillPool = (spillConfig.getNumSpillThreads() <= 0) ? null :
            Executors.newFixedThreadPool(spillConfig.getNumSpillThreads());
    this.maxRowsPerCall = maxRowsPerCall;
    this.constraintEvaluator = constraintEvaluator;
}
 
Example 5
Source Project: pacbot   Source File: AwsS3BucketService.java    License: Apache License 2.0 6 votes vote down vote up
public boolean uploadFile(final AmazonS3 amazonS3, MultipartFile fileToUpload, String s3BucketName, String key) {
	try {
		File file = AdminUtils.convert(fileToUpload);
		long size = fileToUpload.getSize();
		String contentType = fileToUpload.getContentType();
		ObjectMetadata metadata = new ObjectMetadata();
		metadata.setContentType(contentType);
		metadata.setContentLength(size);
		PutObjectRequest putObjectRequest = new PutObjectRequest(s3BucketName, key, file).withCannedAcl(CannedAccessControlList.PublicRead);
		amazonS3.putObject(putObjectRequest);
		return Boolean.TRUE;
	} catch (IOException exception) {
		log.error(UNEXPECTED_ERROR_OCCURRED, exception);
	} 
	return Boolean.FALSE;
}
 
Example 6
Source Project: beam   Source File: S3ReadableSeekableByteChannel.java    License: Apache License 2.0 6 votes vote down vote up
S3ReadableSeekableByteChannel(AmazonS3 amazonS3, S3ResourceId path, S3Options options)
    throws IOException {
  this.amazonS3 = checkNotNull(amazonS3, "amazonS3");
  checkNotNull(path, "path");
  this.options = checkNotNull(options, "options");

  if (path.getSize().isPresent()) {
    contentLength = path.getSize().get();
    this.path = path;

  } else {
    try {
      contentLength =
          amazonS3.getObjectMetadata(path.getBucket(), path.getKey()).getContentLength();
    } catch (AmazonClientException e) {
      throw new IOException(e);
    }
    this.path = path.withSize(contentLength);
  }
}
 
Example 7
Source Project: components   Source File: S3Connection.java    License: Apache License 2.0 6 votes vote down vote up
private static String getEndpoint(AmazonS3 s3client, String bucket) {
    String bucketLocation = null;
    try { 
        bucketLocation = s3client.getBucketLocation(bucket);
    } catch(IllegalArgumentException e) {
        //java.lang.IllegalArgumentException: Cannot create enum from eu-west-2 value!
        String info = e.getMessage();
        if(info == null || info.isEmpty()) {
            throw e;
        }
        Pattern regex = Pattern.compile("[a-zA-Z]+-[a-zA-Z]+-[1-9]");
        Matcher matcher = regex.matcher(info);
        if(matcher.find()) {
            bucketLocation = matcher.group(0);
        } else {
            throw e;
        }
    }
    String region = S3RegionUtil.getBucketRegionFromLocation(bucketLocation);
    return S3RegionUtil.regionToEndpoint(region);
}
 
Example 8
Source Project: emodb   Source File: StashSplitIterator.java    License: Apache License 2.0 6 votes vote down vote up
StashSplitIterator(AmazonS3 s3, String bucket, String key) {
    InputStream rawIn = new RestartingS3InputStream(s3, bucket, key);
    try {
        // File is gzipped
        // Note:
        //   Because the content may be concatenated gzip files we cannot use the default GZIPInputStream.
        //   GzipCompressorInputStream supports concatenated gzip files.
        GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(rawIn, true);
        _in = new BufferedReader(new InputStreamReader(gzipIn, Charsets.UTF_8));
        // Create a line reader
        _reader = new LineReader(_in);
    } catch (Exception e) {
        try {
            Closeables.close(rawIn, true);
        } catch (IOException ignore) {
            // Won't happen, already caught and logged
        }
        throw Throwables.propagate(e);
    }
}
 
Example 9
Source Project: athenz   Source File: S3ChangeLogStore.java    License: Apache License 2.0 6 votes vote down vote up
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("getSignedDomain with S3: {}", domainName);
        }
        
        SignedDomain signedDomain = null;
        try {
            S3Object object = s3.getObject(s3BucketName, domainName);
            try (S3ObjectInputStream s3is = object.getObjectContent()) {
                signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
            }
        } catch (Exception ex) {
            LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain {} error: {}",
                    domainName, ex.getMessage());
        }
        return signedDomain;
    }
 
Example 10
public static <T> T withClientCorrectingRegion(@NotNull final AmazonS3 s3Client,
                                               @NotNull final Map<String, String> settings,
                                               @NotNull final WithS3<T, AmazonS3Exception> withCorrectedClient) {
  try {
    return withCorrectedClient.run(s3Client);
  } catch (AmazonS3Exception awsException) {
    final String correctRegion = extractCorrectedRegion(awsException);
    if (correctRegion != null) {
      LOGGER.debug("Running operation with corrected S3 region [" + correctRegion + "]", awsException);
      final HashMap<String, String> correctedSettings = new HashMap<>(settings);
      correctedSettings.put(REGION_NAME_PARAM, correctRegion);
      return withS3Client(correctedSettings, withCorrectedClient);
    } else {
      throw awsException;
    }
  }
}
 
Example 11
Source Project: java-slack-sdk   Source File: AmazonS3OAuthStateService.java    License: MIT License 6 votes vote down vote up
@Override
public boolean isAvailableInDatabase(String state) {
    AmazonS3 s3 = this.createS3Client();
    S3Object s3Object = getObject(s3, getKey(state));
    if (s3Object == null) {
        return false;
    }
    String millisToExpire = null;
    try {
        millisToExpire = IOUtils.toString(s3Object.getObjectContent());
        return Long.valueOf(millisToExpire) > System.currentTimeMillis();
    } catch (IOException e) {
        log.error("Failed to load a state data for state: {}", state, e);
        return false;
    } catch (NumberFormatException ne) {
        log.error("Invalid state value detected - state: {}, millisToExpire: {}", state, millisToExpire);
        return false;
    }
}
 
Example 12
/**
 * Deletes a regular file.
 */
@Override
public boolean deleteFile(URI target) throws IOException {
	target = target.normalize();
	PooledS3Connection connection = null;
	try {
		connection = connect(target);
		AmazonS3 service = connection.getService();
		String[] path = getPath(target);
		try {
			service.deleteObject(path[0], path[1]);
			return true;
		} catch (AmazonClientException e) {
			throw new IOException(e);
		}
	} finally {
		disconnect(connection);
	}
}
 
Example 13
Source Project: rdf-delta   Source File: S3.java    License: Apache License 2.0 6 votes vote down vote up
public static AmazonS3 buildS3(LocalServerConfig configuration) {
    String region = configuration.getProperty(pRegion);
    String endpoint = configuration.getProperty(pEndpoint);
    String credentialsFile =  configuration.getProperty(pCredentialFile);
    String credentialsProfile =  configuration.getProperty(pCredentialProfile);

    AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
    if ( endpoint == null )
        builder.withRegion(region);
    else  {
        // Needed for S3mock
        builder.withPathStyleAccessEnabled(true);
        builder.withEndpointConfiguration(new EndpointConfiguration(endpoint, region));
        builder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
    }
    if ( credentialsFile != null )
        builder.withCredentials(new ProfileCredentialsProvider(credentialsFile, credentialsProfile));
    return builder.build();
}
 
Example 14
Source Project: halyard   Source File: S3Validator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void validate(ConfigProblemSetBuilder ps, S3PersistentStore n) {
  if (!StringUtils.isEmpty(n.getEndpoint())) {
    return;
  }

  try {
    AWSCredentialsProvider credentialsProvider =
        AwsAccountValidator.getAwsCredentialsProvider(
            n.getAccessKeyId(), secretSessionManager.decrypt(n.getSecretAccessKey()));
    S3Config s3Config = new S3Config();
    S3MetadataStorageProperties s3Properties = new S3MetadataStorageProperties();
    s3Properties.setBucket(n.getBucket());
    s3Properties.setRootFolder(n.getRootFolder());
    s3Properties.setRegion(n.getRegion());
    AmazonS3 s3Client = s3Config.awsS3MetadataClient(credentialsProvider, s3Properties);
    new S3Config().s3StorageService(s3Client, s3Properties);
  } catch (Exception e) {
    ps.addProblem(
        Problem.Severity.ERROR,
        "Failed to ensure the required bucket \""
            + n.getBucket()
            + "\" exists: "
            + e.getMessage());
  }
}
 
Example 15
Source Project: java-slack-sdk   Source File: AmazonS3InstallationServiceTest.java    License: MIT License 6 votes vote down vote up
@Test
public void initializer() {
    AWSCredentials credentials = mock(AWSCredentials.class);
    when(credentials.getAWSAccessKeyId()).thenReturn("valid key");
    AmazonS3 s3 = mock(AmazonS3.class);
    when(s3.doesBucketExistV2(anyString())).thenReturn(true);

    AmazonS3InstallationService service = new AmazonS3InstallationService("test-bucket") {
        @Override
        protected AWSCredentials getCredentials() {
            return credentials;
        }

        @Override
        protected AmazonS3 createS3Client() {
            return s3;
        }
    };
    service.initializer().accept(null);
}
 
Example 16
@Bean
public S3InboundFileSynchronizer s3InboundFileSynchronizer(AmazonS3 amazonS3,
		ResourceIdResolver resourceIdResolver) {
	S3SessionFactory s3SessionFactory = new S3SessionFactory(amazonS3, resourceIdResolver);
	S3InboundFileSynchronizer synchronizer = new S3InboundFileSynchronizer(s3SessionFactory);
	synchronizer.setDeleteRemoteFiles(this.s3SourceProperties.isDeleteRemoteFiles());
	synchronizer.setPreserveTimestamp(this.s3SourceProperties.isPreserveTimestamp());
	String remoteDir = this.s3SourceProperties.getRemoteDir();
	synchronizer.setRemoteDirectory(remoteDir);
	synchronizer.setRemoteFileSeparator(this.s3SourceProperties.getRemoteFileSeparator());
	synchronizer.setTemporaryFileSuffix(this.s3SourceProperties.getTmpFileSuffix());

	if (StringUtils.hasText(this.s3SourceProperties.getFilenamePattern())) {
		synchronizer.setFilter(new S3SimplePatternFileListFilter(this.s3SourceProperties.getFilenamePattern()));
	}
	else if (this.s3SourceProperties.getFilenameRegex() != null) {
		synchronizer.setFilter(new S3RegexPatternFileListFilter(this.s3SourceProperties.getFilenameRegex()));
	}

	return synchronizer;
}
 
Example 17
Source Project: mojito   Source File: S3BlobStorage.java    License: Apache License 2.0 5 votes vote down vote up
public S3BlobStorage(AmazonS3 amazonS3,
                     S3BlobStorageConfigurationProperties s3BlobStorageConfigurationProperties) {
    Preconditions.checkNotNull(amazonS3);
    Preconditions.checkNotNull(s3BlobStorageConfigurationProperties);

    this.amazonS3 = amazonS3;
    this.s3BlobStorageConfigurationProperties = s3BlobStorageConfigurationProperties;
}
 
Example 18
Source Project: front50   Source File: S3ClientFactory.java    License: Apache License 2.0 5 votes vote down vote up
public static AmazonS3 create(
    AWSCredentialsProvider awsCredentialsProvider, S3Properties s3Properties) {
  ClientConfiguration clientConfiguration = new ClientConfiguration();
  if (s3Properties.getProxyProtocol() != null) {
    if (s3Properties.getProxyProtocol().equalsIgnoreCase("HTTPS")) {
      clientConfiguration.setProtocol(Protocol.HTTPS);
    } else {
      clientConfiguration.setProtocol(Protocol.HTTP);
    }
    Optional.ofNullable(s3Properties.getProxyHost()).ifPresent(clientConfiguration::setProxyHost);
    Optional.ofNullable(s3Properties.getProxyPort())
        .map(Integer::parseInt)
        .ifPresent(clientConfiguration::setProxyPort);
  }

  AmazonS3Client client = new AmazonS3Client(awsCredentialsProvider, clientConfiguration);

  if (!StringUtils.isEmpty(s3Properties.getEndpoint())) {
    client.setEndpoint(s3Properties.getEndpoint());

    if (!StringUtils.isEmpty(s3Properties.getRegionOverride())) {
      client.setSignerRegionOverride(s3Properties.getRegionOverride());
    }

    client.setS3ClientOptions(
        S3ClientOptions.builder().setPathStyleAccess(s3Properties.getPathStyleAccess()).build());
  } else {
    Optional.ofNullable(s3Properties.getRegion())
        .map(Regions::fromName)
        .map(Region::getRegion)
        .ifPresent(client::setRegion);
  }

  return client;
}
 
Example 19
Source Project: data-highway   Source File: TruckParkAppIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Primary
@Bean
AmazonS3 testS3(@Value("${s3.port}") int port) {
  return AmazonS3Client
      .builder()
      .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()))
      .withEndpointConfiguration(new EndpointConfiguration("http://127.0.0.1:" + port, "us-west-2"))
      .build();
}
 
Example 20
@Before
public void setupSdk() throws Exception {
	PowerMockito.mockStatic(AWSClientFactory.class);
	AmazonS3 amazonS3 = PowerMockito.mock(AmazonS3.class);
	PowerMockito.when(AWSClientFactory.create(Mockito.any(AwsSyncClientBuilder.class), Mockito.any(StepContext.class)))
			.thenReturn(amazonS3);

	PowerMockito.mockStatic(TransferManagerBuilder.class);
	transferManager = Mockito.mock(TransferManager.class);
	TransferManagerBuilder transferManagerBuilder = PowerMockito.mock(TransferManagerBuilder.class);
	PowerMockito.when(TransferManagerBuilder.standard()).thenReturn(transferManagerBuilder);
	PowerMockito.when(transferManagerBuilder.withS3Client(Mockito.any(AmazonS3.class))).thenReturn(transferManagerBuilder);
	PowerMockito.when(transferManagerBuilder.build()).thenReturn(transferManager);
}
 
Example 21
Source Project: genie   Source File: SimpleStorageRangeResource.java    License: Apache License 2.0 5 votes vote down vote up
SimpleStorageRangeResource(
    final AmazonS3 client,
    final String bucket,
    final String key,
    final String versionId, final TaskExecutor s3TaskExecutor,
    final Pair<Integer, Integer> range
) throws IOException {
    super(client, bucket, key, s3TaskExecutor, versionId);
    this.client =  AmazonS3ProxyFactory.createProxy(client);
    this.bucket = bucket;
    this.key = key;
    this.versionId = versionId;
    this.range = range;

    long tempContentLength = -1;
    try {
        tempContentLength = super.contentLength();
    } catch (FileNotFoundException e) {
        // S3 object does not exist.
        // Upstream code will handle this correctly by checking exists(), contentLength(), etc.
        log.warn("Returning non-existent S3 resource {}/{}", bucket, key);
    }
    this.contentLength = tempContentLength;

    final Integer lower = this.range.getLeft();
    final Integer upper = this.range.getRight();

    if ((lower != null && upper != null && lower > upper)
        || (lower != null && lower > contentLength)) {
        throw new IllegalArgumentException(
            "Invalid range " + lower + "-" + upper + " for S3 object of size " + contentLength
        );
    }
}
 
Example 22
public InventoryReportLineWriter(AmazonS3 client, String destBucketName, String destPrefix,
                                 String srcBucket, InventoryManifest inventoryManifest) throws IOException{
    this.s3Client = client;
    this.bucketName = destBucketName;
    String uuid = UUID.randomUUID().toString();
    this.outputInventoryReportKey = destPrefix + "/" + srcBucket + "/data/" + uuid + ".csv.gz";
    this.schema = CsvSchemaFactory.buildSchema(inventoryManifest);
}
 
Example 23
Source Project: kayenta   Source File: S3StorageService.java    License: Apache License 2.0 5 votes vote down vote up
/** Check to see if the bucket exists, creating it if it is not there. */
public void ensureBucketExists(String accountName) {
  AwsNamedAccountCredentials credentials =
      accountCredentialsRepository.getRequiredOne(accountName);

  AmazonS3 amazonS3 = credentials.getAmazonS3();
  String bucket = credentials.getBucket();
  String region = credentials.getRegion();

  HeadBucketRequest request = new HeadBucketRequest(bucket);

  try {
    amazonS3.headBucket(request);
  } catch (AmazonServiceException e) {
    if (e.getStatusCode() == 404) {
      if (com.amazonaws.util.StringUtils.isNullOrEmpty(region)) {
        log.warn("Bucket {} does not exist. Creating it in default region.", bucket);
        amazonS3.createBucket(bucket);
      } else {
        log.warn("Bucket {} does not exist. Creating it in region {}.", bucket, region);
        amazonS3.createBucket(bucket, region);
      }
    } else {
      log.error("Could not create bucket {}: {}", bucket, e);
      throw e;
    }
  }
}
 
Example 24
Source Project: camel-kafka-connector   Source File: AWSClientUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static AmazonS3 newS3Client() {
    LOG.debug("Creating a new S3 client");
    AmazonS3ClientBuilder clientBuilder = AmazonS3ClientBuilder.standard();

    String awsInstanceType = System.getProperty("aws-service.instance.type");
    String region = getRegion();

    if (awsInstanceType == null || awsInstanceType.equals("local-aws-container")) {
        String amazonHost = System.getProperty(AWSConfigs.AMAZON_AWS_HOST);
        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setProtocol(Protocol.HTTP);

        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(amazonHost, region))
                .withClientConfiguration(clientConfiguration)
                .withCredentials(new TestAWSCredentialsProvider("accesskey", "secretkey"));
    } else {
        clientBuilder
                .withRegion(region)
                .withCredentials(new TestAWSCredentialsProvider());
    }

    clientBuilder
            .withPathStyleAccessEnabled(true);

    return clientBuilder.build();
}
 
Example 25
private static IMultiReportOutput createS3Output() {
	AmazonS3 s3Client = AmazonS3ClientBuilder.standard() //
			.withRegion("us-east-1") //
			.withCredentials(new SystemPropertiesCredentialsProvider()) //
			.build();
	return new S3MultiReportOutput(s3Client, "download.eclipselab.org", "jdkdiff/");
}
 
Example 26
Source Project: aws-big-data-blog   Source File: MCAWS.java    License: Apache License 2.0 5 votes vote down vote up
public static void putImageS3(String bucketName, String key, String fileName) {
    AmazonS3 s3 = new AmazonS3Client();
    Region usWest2 = Region.getRegion(Regions.US_WEST_2);
    //Region usWest2 = Region.getRegion(s3Region);
    s3.setRegion(usWest2);
    try {
        File file = new File(fileName);
        s3.putObject(new PutObjectRequest(bucketName, key, file));
    } catch (Exception e) { System.out.println("ERROR ON IMAGE FILE"); }
}
 
Example 27
Source Project: kangaroo   Source File: S3OptimizedFileInputFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected List<FileStatus> listStatus(final JobContext job) throws IOException {
    final Path[] dirs = getInputPaths(job);
    if (dirs.length == 0) {
        throw new IOException("No input paths specified in job");
    }
    final long blockSize = job.getConfiguration().getLong(S3NativeFileSystemConfigKeys.S3_NATIVE_BLOCK_SIZE_KEY,
            S3NativeFileSystemConfigKeys.S3_NATIVE_BLOCK_SIZE_DEFAULT);
    final AmazonS3 s3Client = S3HadoopUtils.getS3Client(job.getConfiguration());
    return S3InputFormatUtils.getFileStatuses(s3Client, blockSize, dirs);
}
 
Example 28
@NotNull
private Integer deleteChunk(@NotNull final String pathPrefix,
                            @NotNull final String bucketName,
                            @NotNull final AmazonS3 client,
                            @NotNull final List<String> part,
                            @NotNull final Supplier<String> info) throws Exception {
  final List<DeleteObjectsRequest.KeyVersion> objectKeys = part.stream().map(path -> new DeleteObjectsRequest.KeyVersion(pathPrefix + path)).collect(Collectors.toList());
  final DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName).withKeys(objectKeys);
  return executeWithNewThreadName(info.get(),
                                  () -> doUnderContextClassLoader(S3Util.class.getClassLoader(), () -> client.deleteObjects(deleteObjectsRequest).getDeletedObjects().size()));
}
 
Example 29
Source Project: nexus-public   Source File: MultipartUploader.java    License: Eclipse Public License 1.0 5 votes vote down vote up
@Override
public void upload(final AmazonS3 s3, final String bucket, final String key, final InputStream contents) {
  try (InputStream input = contents) {
    InputStream chunkOne = readChunk(input);
    if (chunkOne.available() < chunkSize) {
      uploadSinglePart(s3, bucket, key, chunkOne);
    }
    else {
      uploadMultiPart(s3, bucket, key, chunkOne, contents);
    }
  }
  catch(IOException | SdkClientException e) { // NOSONAR
    throw new BlobStoreException("Error uploading blob", e, null);
  }
}
 
Example 30
Source Project: ReCiter   Source File: AmazonS3Config.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return AmazonS3 client object
 */
@Bean
@Scope("singleton")
public AmazonS3 amazonS3() {
	
	if(isS3Use && !isDynamoDbLocal) {
 	final AmazonS3 s3 = AmazonS3ClientBuilder
 						.standard()
 						.withCredentials(new AWSStaticCredentialsProvider(new AWSCredentials() {
					
					@Override
					public String getAWSSecretKey() {
						return amazonAWSSecretKey;
					}
					
					@Override
					public String getAWSAccessKeyId() {
						return amazonAWSAccessKey;
					}
				}))
 						.withRegion(awsS3Region)
 						.build();
 	createBucket(s3);
 	return s3;
	}
						
	return null;		
}