Java Code Examples for com.amazonaws.services.s3.model.S3Object

The following examples show how to use com.amazonaws.services.s3.model.S3Object. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: vividus   Source File: S3BucketStepsTests.java    License: Apache License 2.0 6 votes vote down vote up
@Test
void fetchCsvFileTest() throws IOException
{
    String objectKey = S3_OBJECT_KEY + ".csv";
    byte[] csv = ResourceUtils.loadResourceAsByteArray(CSV_FILE_PATH);

    S3Object s3Object = mock(S3Object.class);
    S3ObjectInputStream s3ObjectInputStream = new S3ObjectInputStream(new ByteArrayInputStream(csv), null);
    when(s3Object.getObjectContent()).thenReturn(s3ObjectInputStream);
    when(amazonS3Client.getObject(S3_BUCKET_NAME, objectKey)).thenReturn(s3Object);

    Set<VariableScope> scopes = Set.of(VariableScope.SCENARIO);
    String variableName = "varName";
    steps.fetchCsvObject(objectKey, S3_BUCKET_NAME, scopes, variableName);
    verify(amazonS3Client).getObject(S3_BUCKET_NAME, objectKey);
    verify(bddVariableContext).putVariable(scopes, variableName, List.of(Map.of("id", "1")));
}
 
Example 2
Source Project: spring-s3-properties-loader   Source File: S3Service.java    License: MIT License 6 votes vote down vote up
/**
 * @param bucketName + key location
 * @return {@link S3Object} for the given aws s3 location.
 * @throws InvalidS3LocationException for invalid location params
 * @throws S3ResourceException for connection and availability errors
 */
public S3Object retriveFrom(String location) {
	if (isEmpty(location)) {
		throw new InvalidS3LocationException("Location cannot be empty or null");
	}

	String path = location.startsWith(S3_PROTOCOL_PREFIX) ? location.substring(S3_PROTOCOL_PREFIX.length(), location.length()) : location;

	if(!path.contains("/")) {
		throw new InvalidS3LocationException("The location must contains the full path of the properties file");
	}

	String bucketName = path.substring(0, path.indexOf('/'));
	String keyName = path.substring(path.indexOf('/') + 1);

	try {
		return amazonS3.getObject(bucketName, keyName);
	} catch (Exception e) {
		throw new S3ResourceException("Could not load resource from " + location, e);
	}
}
 
Example 3
Source Project: java-slack-sdk   Source File: AmazonS3OAuthStateService.java    License: MIT License 6 votes vote down vote up
@Override
public boolean isAvailableInDatabase(String state) {
    AmazonS3 s3 = this.createS3Client();
    S3Object s3Object = getObject(s3, getKey(state));
    if (s3Object == null) {
        return false;
    }
    String millisToExpire = null;
    try {
        millisToExpire = IOUtils.toString(s3Object.getObjectContent());
        return Long.valueOf(millisToExpire) > System.currentTimeMillis();
    } catch (IOException e) {
        log.error("Failed to load a state data for state: {}", state, e);
        return false;
    } catch (NumberFormatException ne) {
        log.error("Invalid state value detected - state: {}, millisToExpire: {}", state, millisToExpire);
        return false;
    }
}
 
Example 4
@BeforeClass
public static void setup() throws IOException {
	File remoteFolder = TEMPORARY_FOLDER.newFolder("remote");

	File aFile = new File(remoteFolder, "1.test");
	FileCopyUtils.copy("Hello".getBytes(), aFile);
	File bFile = new File(remoteFolder, "2.test");
	FileCopyUtils.copy("Bye".getBytes(), bFile);
	File otherFile = new File(remoteFolder, "otherFile");
	FileCopyUtils.copy("Other\nOther2".getBytes(), otherFile);

	S3_OBJECTS = new ArrayList<>();

	for (File file : remoteFolder.listFiles()) {
		S3Object s3Object = new S3Object();
		s3Object.setBucketName(S3_BUCKET);
		s3Object.setKey(file.getName());
		s3Object.setObjectContent(new FileInputStream(file));
		S3_OBJECTS.add(s3Object);
	}

	String localFolder = TEMPORARY_FOLDER.newFolder("local").getAbsolutePath();

	System.setProperty("s3.localDir", localFolder);
}
 
Example 5
Source Project: singleton   Source File: S3OneComponentDaoImpl.java    License: Eclipse Public License 2.0 6 votes vote down vote up
/**
 * get one component bundle files from s3 server as json String
 */
@Override
public String get2JsonStr(String productName, String version, String component, String locale)
      throws DataException {
   String filePath = S3Utils.genProductVersionS3Path(productName, version) + component
         + ConstantsChar.BACKSLASH + ResourceFilePathGetter.getLocalizedJSONFileName(locale);
   String result = null;
   if (s3Client.getS3Client().doesObjectExist(config.getBucketName(), filePath)) {
      S3Object o = s3Client.getS3Client().getObject(config.getBucketName(), filePath);
      if (o != null) {
         try {
            result = S3Utils.convertS3Obj2Str(o);
         } catch (IOException e) {
            logger.warn(e.getMessage(), e);
            throw new DataException(S3_NOT_EXIST_STR + filePath);
         }
      } else {
         throw new DataException(S3_NOT_EXIST_STR + filePath);
      }
   }
   if (result == null) {
      throw new DataException(S3_NOT_EXIST_STR + filePath);
   }
   return result;
}
 
Example 6
Source Project: singleton   Source File: S3Utils.java    License: Eclipse Public License 2.0 6 votes vote down vote up
/**
 * convert the S3 Object to String
 */
public static String convertS3Obj2Str(S3Object s3Obj) throws IOException {
   S3ObjectInputStream s3is = s3Obj.getObjectContent();
   ByteArrayOutputStream fos = new ByteArrayOutputStream();
   byte[] read_buf = new byte[1024];
   int read_len = 0;
   try {
      while ((read_len = s3is.read(read_buf)) > 0) {
         fos.write(read_buf, 0, read_len);
      }
      return fos.toString(ConstantsUnicode.UTF8);
   } finally {
      s3is.close();
      fos.close();

   }
}
 
Example 7
Source Project: herd   Source File: S3DaoImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Properties getProperties(String bucketName, String key, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto)
{
    AmazonS3Client s3Client = getAmazonS3(s3FileTransferRequestParamsDto);

    try
    {
        S3Object s3Object = getS3Object(s3Client, bucketName, key, true);
        return javaPropertiesHelper.getProperties(s3Object.getObjectContent());
    }
    catch (IllegalArgumentException e)
    {
        throw new IllegalArgumentException("The properties file in S3 bucket '" + bucketName + "' and key '" + key + "' is invalid.", e);
    }
    finally
    {
        s3Client.shutdown();
    }
}
 
Example 8
Source Project: ReCiter   Source File: DynamoDbS3Operations.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This function retrieves large object from S3
 * @param bucketName
 * @param keyName
 * @param objectClass
 * @return
 */
public <T> Object retrieveLargeItem(String bucketName, String keyName, Class<T> objectClass) {
	try {
		S3Object s3Object = s3.getObject(new GetObjectRequest(bucketName.toLowerCase(), keyName));
		String objectContent = IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8);
		if(objectClass == ReCiterFeature.class) {
			ReCiterFeature reCiterFeature = OBJECT_MAPPER.readValue(objectContent, ReCiterFeature.class);
			return reCiterFeature;
		}
		
	} catch (IOException | AmazonServiceException e) {
		log.error(e.getMessage());
	}
	return null;
	
}
 
Example 9
Source Project: pacbot   Source File: ErrorManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+dataSource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 
Example 10
Source Project: athenz   Source File: S3ChangeLogStore.java    License: Apache License 2.0 6 votes vote down vote up
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("getSignedDomain with S3: {}", domainName);
        }
        
        SignedDomain signedDomain = null;
        try {
            S3Object object = s3.getObject(s3BucketName, domainName);
            try (S3ObjectInputStream s3is = object.getObjectContent()) {
                signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
            }
        } catch (Exception ex) {
            LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain {} error: {}",
                    domainName, ex.getMessage());
        }
        return signedDomain;
    }
 
Example 11
Source Project: ats-framework   Source File: S3Operations.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get MD5, size, owner, storage class and last modification time for a desired file in the pointed bucket
 *
 * @param fileName the file name
 */
@PublicAtsApi
public S3ObjectInfo getFileMetadata( String fileName ) {

    try {
        S3Object element = s3Client.getObject(bucketName, fileName);
        if (element != null) {
            ObjectMetadata metaData = element.getObjectMetadata();
            S3ObjectInfo s3Info = new S3ObjectInfo();
            s3Info.setBucketName(fileName);
            s3Info.setLastModified(metaData.getLastModified());
            s3Info.setMd5(metaData.getETag());
            s3Info.setName(element.getKey());
            s3Info.setSize(metaData.getContentLength());

            return s3Info;
        } else {
            throw new NoSuchElementException("File with name '" + fileName + "' does not exist!");
        }
    } catch (Exception e) {
        handleExeption(e, "Could not retrieve metadata for S3 object with key '" + fileName + "'");
    }
    return null;
}
 
Example 12
private static void streamReadAndDownloadObject(
        final File workspace,
        final S3Object sessionObject,
        final String downloadedFileName) throws IOException {

    final File outputFile = new File(workspace, downloadedFileName);

    try (final S3ObjectInputStream objectContents = sessionObject.getObjectContent();
         final OutputStream outputStream = new FileOutputStream(outputFile)) {
        final int BUFFER_SIZE = 8192;
        final byte[] buffer = new byte[BUFFER_SIZE];

        int i;
        while ((i = objectContents.read(buffer)) != -1) {
            outputStream.write(buffer, 0, i);
        }
    }
}
 
Example 13
Source Project: s3proxy   Source File: AwsSdkAnonymousTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAwsV4SignatureChunkedAnonymous() throws Exception {
    client = AmazonS3ClientBuilder.standard()
        .withChunkedEncodingDisabled(false)
        .withEndpointConfiguration(s3EndpointConfig)
        .build();

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, "foo", BYTE_SOURCE.openStream(),
            metadata);

    S3Object object = client.getObject(containerName, "foo");
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
        InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 
Example 14
private void downloadAndExtract(
        final S3Object sessionObject,
        final File workspace,
        final String downloadedFileName,
        final TaskListener listener) throws IOException {

    downloadArtifacts(sessionObject, workspace, downloadedFileName, listener);

    final File fullFilePath = new File(workspace, downloadedFileName);

    try {
        ExtractionTools.decompressFile(fullFilePath, workspace, model.getCompressionType(), listener);
        LoggingHelper.log(listener, "Artifact uncompressed successfully");
    } finally {
        if (fullFilePath != null) {
            try {
                ExtractionTools.deleteTemporaryCompressedFile(fullFilePath);
            } catch (final IOException ex) {
                LoggingHelper.log(listener, "Could not delete temporary file: %s", ex.getMessage());
                LoggingHelper.log(listener, ex);
            }
        }
    }
}
 
Example 15
@Test
public void loadGetsObjectsAndReturnsTrueIfItExistsInS3() throws Exception {
  /** Setup **/
  buildCacheService = new AwsS3BuildCacheService(s3, "bucketName", null, true);
  doReturn(true).when(s3).doesObjectExist("bucketName", "abcdefghijkl123456789");
  S3Object s3Object = mock(S3Object.class);
  doReturn(s3Object).when(s3).getObject("bucketName", "abcdefghijkl123456789");
  S3ObjectInputStream s3ObjectInputStream = mock(S3ObjectInputStream.class);
  doReturn(s3ObjectInputStream).when(s3Object).getObjectContent();

  /** Run **/
  boolean result = buildCacheService.load(key, reader);

  /** Check **/
  assertTrue(result);
  verify(reader).readFrom(s3ObjectInputStream);
}
 
Example 16
public static void downloadFromS3(String s3BucketName, List<String> objectKeys,
                                  String downloadDirectory) {

    // Initializes the Amazon S3 client.
    AmazonS3 s3Client = AmazonS3ClientBuilder.defaultClient();

    try {
        // Downloads each object to the specified file path.
        for (String key : objectKeys) {
            S3Object object = s3Client.getObject(s3BucketName, key);
            String endpointsFileName = key.substring(key.lastIndexOf("/"));
            Path filePath = Paths.get(downloadDirectory + endpointsFileName);

            System.out.format("Downloading %s to %s . . .\n",
                    filePath.getFileName(), filePath.getParent());

            writeObjectToFile(filePath, object);
        }
        System.out.println("Download finished.");
    } catch (AmazonServiceException | NullPointerException e) {
        System.err.println(e.getMessage());
        System.exit(1);
    }

}
 
Example 17
Source Project: circus-train   Source File: S3S3CopierTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void copyOneFile() throws Exception {
  client.putObject("source", "data", inputData);

  Path sourceBaseLocation = new Path("s3://source/data");
  Path replicaLocation = new Path("s3://target/data2");
  List<Path> sourceSubLocations = new ArrayList<>();
  S3S3Copier s3s3Copier = newS3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation);
  Metrics metrics = s3s3Copier.copy();
  assertThat(metrics.getBytesReplicated(), is(7L));
  assertThat(metrics.getMetrics().get(S3S3CopierMetrics.Metrics.TOTAL_BYTES_TO_REPLICATE.name()), is(7L));
  S3Object object = client.getObject("target", "data2");
  String data = IOUtils.toString(object.getObjectContent());
  assertThat(data, is("bar foo"));
  assertThat(registry.getGauges().containsKey(RunningMetrics.S3S3_CP_BYTES_REPLICATED.name()), is(true));
}
 
Example 18
@Test
void getInputStream_existingObject_returnsInputStreamWithContent() throws Exception {
	// Arrange
	AmazonS3 amazonS3 = mock(AmazonS3.class);
	ObjectMetadata objectMetadata = mock(ObjectMetadata.class);
	when(amazonS3.getObjectMetadata(any(GetObjectMetadataRequest.class)))
			.thenReturn(objectMetadata);

	S3Object s3Object = new S3Object();
	s3Object.setObjectMetadata(objectMetadata);
	s3Object.setObjectContent(new ByteArrayInputStream(new byte[] { 42 }));
	when(amazonS3.getObject(any(GetObjectRequest.class))).thenReturn(s3Object);

	// Act
	SimpleStorageResource simpleStorageResource = new SimpleStorageResource(amazonS3,
			"bucket", "object", new SyncTaskExecutor());

	// Assert
	assertThat(simpleStorageResource.exists()).isTrue();
	assertThat(simpleStorageResource.getInputStream().read()).isEqualTo(42);
}
 
Example 19
Source Project: attic-apex-malhar   Source File: S3BlockReader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * S3 block read would be achieved through the AmazonS3 client. Following are the steps to achieve:
 * (1) Create the objectRequest from bucketName and filePath.
 * (2) Set the range to the above created objectRequest.
 * (3) Get the object portion through AmazonS3 client API.
 * (4) Get the object content from the above object portion.
 * @return the block entity
 * @throws IOException
 */
@Override
protected Entity readEntity() throws IOException
{
  entity.clear();
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(
      bucketName, filePath);
  rangeObjectRequest.setRange(offset, blockMetadata.getLength() - 1);
  S3Object objectPortion = s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  byte[] record = ByteStreams.toByteArray(wrappedStream);
  entity.setUsedBytes(record.length);
  entity.setRecord(record);
  wrappedStream.close();
  return entity;
}
 
Example 20
Source Project: s3proxy   Source File: AwsSdkTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAtomicMpuAbort() throws Exception {
    String key = "testAtomicMpuAbort";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, key, BYTE_SOURCE.openStream(),
            metadata);

    InitiateMultipartUploadRequest initRequest =
            new InitiateMultipartUploadRequest(containerName, key);
    InitiateMultipartUploadResult initResponse =
            client.initiateMultipartUpload(initRequest);
    String uploadId = initResponse.getUploadId();

    client.abortMultipartUpload(new AbortMultipartUploadRequest(
                containerName, key, uploadId));

    S3Object object = client.getObject(containerName, key);
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 
Example 21
Source Project: incubator-gobblin   Source File: AWSSdkClient.java    License: Apache License 2.0 6 votes vote down vote up
/***
 * Download a S3 object to local directory
 *
 * @param s3ObjectSummary S3 object summary for the object to download
 * @param targetDirectory Local target directory to download the object to
 * @throws IOException If any errors were encountered in downloading the object
 */
public void downloadS3Object(S3ObjectSummary s3ObjectSummary,
    String targetDirectory)
    throws IOException {

  final AmazonS3 amazonS3 = getS3Client();

  final GetObjectRequest getObjectRequest = new GetObjectRequest(
      s3ObjectSummary.getBucketName(),
      s3ObjectSummary.getKey());

  final S3Object s3Object = amazonS3.getObject(getObjectRequest);

  final String targetFile = StringUtils.removeEnd(targetDirectory, File.separator) + File.separator + s3Object.getKey();
  FileUtils.copyInputStreamToFile(s3Object.getObjectContent(), new File(targetFile));

  LOGGER.info("S3 object downloaded to file: " + targetFile);
}
 
Example 22
/**
 * Reads a spilled block.
 *
 * @param spillLocation The location to read the spilled Block from.
 * @param key The encryption key to use when reading the spilled Block.
 * @param schema The Schema to use when deserializing the spilled Block.
 * @return The Block stored at the spill location.
 */
protected Block read(S3SpillLocation spillLocation, EncryptionKey key, Schema schema)
{
    try {
        logger.debug("write: Started reading block from S3");
        S3Object fullObject = amazonS3.getObject(spillLocation.getBucket(), spillLocation.getKey());
        logger.debug("write: Completed reading block from S3");
        Block block = blockCrypto.decrypt(key, ByteStreams.toByteArray(fullObject.getObjectContent()), schema);
        logger.debug("write: Completed decrypting block of size.");
        return block;
    }
    catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}
 
Example 23
Source Project: presto   Source File: S3TableConfigClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map.
 */
private void updateTablesFromS3()
{
    long now = System.currentTimeMillis();

    AmazonS3Client s3client = clientManager.getS3Client();

    for (S3ObjectSummary summary : getObjectSummaries()) {
        if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) {
            // New or updated file, so we must read from AWS
            if (summary.getKey().endsWith("/")) {
                continue;
            }

            log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey());
            S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey()));

            try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) {
                KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader));
                descriptors.put(summary.getKey(), table);
                log.info("Put table description into the map from %s", summary.getKey());
            }
            catch (IOException iox) {
                log.error("Problem reading input stream from object.", iox);
                throwIfUnchecked(iox);
                throw new RuntimeException(iox);
            }
        }
    }

    log.info("Completed updating table definitions from S3.");
    lastCheck = now;
}
 
Example 24
Source Project: presto   Source File: MockAmazonS3.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
{
    if (getObjectHttpCode != HTTP_OK) {
        AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode);
        exception.setStatusCode(getObjectHttpCode);
        throw exception;
    }
    return null;
}
 
Example 25
Source Project: pentaho-kettle   Source File: S3ObjectsProviderTest.java    License: Apache License 2.0 5 votes vote down vote up
private static S3Object buildS3Object( Bucket bucket, String key, String dataString ) {
  S3Object s3Object = new S3Object();
  s3Object.setKey( key );
  s3Object.setBucketName( bucket.getName() );
  s3Object.setObjectContent( new ByteArrayInputStream( dataString.getBytes() ) );
  return s3Object;
}
 
Example 26
Source Project: datacollector   Source File: TestAmazonS3Executor.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyObject() throws Exception {
  String newName = UUID.randomUUID().toString();

  AmazonS3ExecutorConfig config = getConfig();
  config.taskConfig.taskType = TaskType.COPY_OBJECT;
  config.taskConfig.copyTargetLocation = newName;

  AmazonS3Executor executor = new AmazonS3Executor(config);
  TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
    .build();
  runner.runInit();

  try {
    s3client.putObject(new PutObjectRequest(BUCKET_NAME, objectName, IOUtils.toInputStream("content"), new ObjectMetadata()));
    runner.runWrite(ImmutableList.of(getTestRecord()));

    S3Object object = s3client.getObject(BUCKET_NAME, newName);
    S3ObjectInputStream objectContent = object.getObjectContent();

    List<String> stringList = IOUtils.readLines(objectContent);
    Assert.assertEquals(1, stringList.size());
    Assert.assertEquals("content", stringList.get(0));

    Assert.assertTrue(s3client.doesObjectExist(BUCKET_NAME, objectName));

    Assert.assertEquals(1, runner.getEventRecords().size());
    assertEvent(runner.getEventRecords().get(0), newName);
  } finally {
    runner.runDestroy();
  }
}
 
Example 27
Source Project: XRTB   Source File: SimpleSet.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * A simple set. Reads S3 object and put into the Set.
 * @param name String. The name of the object.
 * @param object S3Object. The S3 object to read.
 * @throws Exception on S3 or I/O options.
 */
public SimpleSet(String name, S3Object object) throws Exception {
	InputStream objectData = object.getObjectContent();
	BufferedReader br=new BufferedReader(new InputStreamReader(objectData));
	message = "Initialize Simple Membership: " + object.getBucketName() + " as " + name;
	makeSet(br);
	
	symbols.put(name, this);
}
 
Example 28
Source Project: hop   Source File: S3FileObjectTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetS3Object() throws Exception {
  when( s3ServiceMock.getObject( anyString(), anyString() ) ).thenReturn( new S3Object() );
  S3FileObject s3FileObject = new S3FileObject( filename, fileSystemSpy );
  S3Object s3Object = s3FileObject.getS3Object();
  assertNotNull( s3Object );
}
 
Example 29
Source Project: openbd-core   Source File: Read.java    License: GNU General Public License v3.0 5 votes vote down vote up
private cfData	readToMemory(AmazonS3 s3Client, String bucket, String key, String aes256key, int retry, int retryseconds) throws Exception {
	
	// Let us run around the number of attempts
	int attempts = 0;
	while ( attempts < retry ){
		try{
			
			GetObjectRequest gor = new GetObjectRequest(bucket, key);
			if ( aes256key != null && !aes256key.isEmpty() )
				gor.setSSECustomerKey( new SSECustomerKey(aes256key) );
	
			S3Object s3object = s3Client.getObject(gor);
			String contentType = s3object.getObjectMetadata().getContentType();
			
			ByteArrayOutputStream	baos	= new ByteArrayOutputStream( 32000 );
			StreamUtil.copyTo(s3object.getObjectContent(), baos, false );
			
			if ( contentType.indexOf("text") != -1 || contentType.indexOf("javascript") != -1 ){
				return new cfStringData( baos.toString() );
			}else{
				return new cfBinaryData( baos.toByteArray() );
			}
			
		}catch(Exception e){
			cfEngine.log("Failed: AmazonS3Read(bucket=" + bucket + "; key=" + key + "; attempt=" + (attempts+1) + "; exception=" + e.getMessage() + ")");
			attempts++;
	
			if ( attempts == retry )
				throw e;
			else
				Thread.sleep( retryseconds*1000 );
		}
	}
	
	return null; // should never 
}
 
Example 30
Source Project: iaf   Source File: AmazonS3FileSystemTestHelper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public InputStream _readFile(String folder, String filename) throws FileNotFoundException {
	final S3Object file = s3Client.getObject(bucketName, filename);
	InputStream is = file.getObjectContent();
	FilterInputStream fos = new FilterInputStream(is) {
		@Override
		public void close() throws IOException {
			super.close();
			file.close();
		}
	};

	return fos;
}