com.amazonaws.services.s3.model.S3ObjectInputStream Java Examples

The following examples show how to use com.amazonaws.services.s3.model.S3ObjectInputStream. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3ObjectSerializer.java    From camel-kafka-connector with Apache License 2.0 7 votes vote down vote up
@Override
public byte[] serialize(String topic, S3ObjectInputStream data) {
    InputStream is = data.getDelegateStream();
    ByteArrayOutputStream buffer = new ByteArrayOutputStream();

    int nRead;
    byte[] byteArray = new byte[16384];

    try {
        while ((nRead = is.read(byteArray, 0, byteArray.length)) != -1) {
            buffer.write(byteArray, 0, nRead);
        }
    } catch (IOException e) {
        LOG.warn("I/O error while serializing data from or to topic {}: {} | {}", topic, e.getMessage(), e);
    }

    return buffer.toByteArray();
}
 
Example #2
Source File: ContentHelperTest.java    From aws-photosharing-example with Apache License 2.0 6 votes vote down vote up
@Test
public void contentTest() throws Exception {

    URL url = this.getClass().getResource("../../../../amazon-aws-logo.jpg");
    String tmpFileName = url.getFile();

    File file = new File(tmpFileName);
    String fileName = file.getName();
    InputStream is = url.openStream();
    String contentType = URLConnection.guessContentTypeFromStream(is);

    contentHelper.uploadContent(contentType, file.length(), bucketName, fileName, is);

    Thread.sleep(500);
    boolean doesObjectExist = s3Client.doesObjectExist(bucketName, fileName);
    Assert.assertTrue(doesObjectExist);

    S3ObjectInputStream inputStream = contentHelper.downloadContent(bucketName, fileName);
    Assert.assertNotNull(inputStream);

    contentHelper.deleteContent(bucketName, fileName);
    Thread.sleep(500);

    doesObjectExist = s3Client.doesObjectExist(bucketName, fileName);
    Assert.assertFalse(doesObjectExist);
}
 
Example #3
Source File: S3ChangeLogStore.java    From athenz with Apache License 2.0 6 votes vote down vote up
SignedDomain getSignedDomain(AmazonS3 s3, String domainName) {

        if (LOGGER.isDebugEnabled()) {
            LOGGER.debug("getSignedDomain with S3: {}", domainName);
        }
        
        SignedDomain signedDomain = null;
        try {
            S3Object object = s3.getObject(s3BucketName, domainName);
            try (S3ObjectInputStream s3is = object.getObjectContent()) {
                signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
            }
        } catch (Exception ex) {
            LOGGER.error("AWSS3ChangeLog: getSignedDomain - unable to get domain {} error: {}",
                    domainName, ex.getMessage());
        }
        return signedDomain;
    }
 
Example #4
Source File: S3ChangeLogStore.java    From athenz with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
    SignedDomain signedDomain = null;
    try {
        S3Object object = s3.getObject(s3BucketName, domainName);
        try (S3ObjectInputStream s3is = object.getObjectContent()) {
            signedDomain = jsonMapper.readValue(s3is, SignedDomain.class);
        }
    } catch (Exception ex) {
        LOGGER.error("AWSS3ChangeLogThread: ObjectS3Thread- getSignedDomain - unable to get domain {} error: {}",
                domainName, ex.getMessage());
    }
    if (signedDomain != null) {
        signedDomainMap.put(domainName, signedDomain);
    }
}
 
Example #5
Source File: DownloadCallable.java    From aws-codepipeline-plugin-for-jenkins with Apache License 2.0 6 votes vote down vote up
private static void streamReadAndDownloadObject(
        final File workspace,
        final S3Object sessionObject,
        final String downloadedFileName) throws IOException {

    final File outputFile = new File(workspace, downloadedFileName);

    try (final S3ObjectInputStream objectContents = sessionObject.getObjectContent();
         final OutputStream outputStream = new FileOutputStream(outputFile)) {
        final int BUFFER_SIZE = 8192;
        final byte[] buffer = new byte[BUFFER_SIZE];

        int i;
        while ((i = objectContents.read(buffer)) != -1) {
            outputStream.write(buffer, 0, i);
        }
    }
}
 
Example #6
Source File: MainActivity.java    From aws-iotbot with Apache License 2.0 6 votes vote down vote up
public void GetFromS3(){
    new Thread(new Runnable() {
        @Override
        public void run() {
            try {
                S3ObjectInputStream content = s3Client.getObject(BUCKET, KEY).getObjectContent();
                byte[] bytes = IOUtils.toByteArray(content);
                bitmap = BitmapFactory.decodeByteArray(bytes, 0, bytes.length);
                setBitmap(bitmap);
                runOnUiThread(new Runnable() {
                    @Override
                    public void run() {
                        imageFromS3.setScaleType(ImageView.ScaleType.FIT_XY);
                        imageFromS3.setImageBitmap(getBitmap());
                    }
                });
            } catch (Exception e) {
                Log.e(LOG_TAG,
                        "Exception occurred when retrieving image from S3",
                        e);
            }
        }
    }).start();
}
 
Example #7
Source File: S3RecordReader.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
/**
 * S3 block read would be achieved through the AmazonS3 client. Following
 * are the steps to achieve: (1) Create the objectRequest from bucketName
 * and filePath. (2) Set the range to the above created objectRequest. (3)
 * Get the object portion through AmazonS3 client API. (4) Get the object
 * content from the above object portion.
 *
 * @param bytesFromCurrentOffset
 *          bytes read till now from current offset
 * @param bytesToFetch
 *          the number of bytes to be fetched
 * @return the number of bytes read, -1 if 0 bytes read
 * @throws IOException
 */

@Override
protected int readData(final long bytesFromCurrentOffset, final int bytesToFetch) throws IOException
{
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(s3Params.bucketName, s3Params.filePath);
  rangeObjectRequest.setRange(offset + bytesFromCurrentOffset, offset + bytesFromCurrentOffset + bytesToFetch - 1);
  S3Object objectPortion = s3Params.s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  buffer = ByteStreams.toByteArray(wrappedStream);
  wrappedStream.close();
  int bufferLength = buffer.length;
  if (bufferLength <= 0) {
    return -1;
  }
  return bufferLength;
}
 
Example #8
Source File: pinpoint_export_endpoints.java    From aws-doc-sdk-examples with Apache License 2.0 6 votes vote down vote up
private static void writeObjectToFile(Path filePath, S3Object object) {

        // Writes the contents of the S3 object to a file.
        File endpointsFile = new File(filePath.toAbsolutePath().toString());
        try (FileOutputStream fos = new FileOutputStream(endpointsFile);
             S3ObjectInputStream s3is = object.getObjectContent()) {
            byte[] read_buf = new byte[1024];
            int read_len = 0;
            while ((read_len = s3is.read(read_buf)) > 0) {
                fos.write(read_buf, 0, read_len);
            }
        } catch (IOException e) {
            System.err.println(e.getMessage());
            System.exit(1);
        }
    }
 
Example #9
Source File: S3BlockReader.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
/**
 * S3 block read would be achieved through the AmazonS3 client. Following are the steps to achieve:
 * (1) Create the objectRequest from bucketName and filePath.
 * (2) Set the range to the above created objectRequest.
 * (3) Get the object portion through AmazonS3 client API.
 * (4) Get the object content from the above object portion.
 * @return the block entity
 * @throws IOException
 */
@Override
protected Entity readEntity() throws IOException
{
  entity.clear();
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(
      bucketName, filePath);
  rangeObjectRequest.setRange(offset, blockMetadata.getLength() - 1);
  S3Object objectPortion = s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  byte[] record = ByteStreams.toByteArray(wrappedStream);
  entity.setUsedBytes(record.length);
  entity.setRecord(record);
  wrappedStream.close();
  return entity;
}
 
Example #10
Source File: InventoryManifestRetrieverTest.java    From s3-inventory-usage-examples with Apache License 2.0 6 votes vote down vote up
@Test (expected = ChecksumMismatchException.class)
public void getInventoryManifestMD5Mismatch() throws Exception {
    InventoryManifest expectedManifest = manifest();
    byte[] expectedManifestBytes = manifestBytes(expectedManifest);
    byte[] errorBytes = "ERROR".getBytes();
    byte[] wrongManifestBytes = ArrayUtils.addAll(expectedManifestBytes, errorBytes);
    when(mockS3JsonObject.getObjectContent()).thenReturn(new S3ObjectInputStream(
            new ByteArrayInputStream(wrongManifestBytes), null));
    String expectedChecksum = "37289f10a76751046658f6c5e0ab41d9";

    byte[] expectedChecksumBytes = expectedChecksum.getBytes(StandardCharsets.UTF_8);
    when(mockS3ChecksumObject.getObjectContent()).thenReturn(new S3ObjectInputStream(
            new ByteArrayInputStream(expectedChecksumBytes), null));
    when(mockS3Client.getObject(getObjectRequestCaptor.capture())).
            thenReturn(mockS3JsonObject)
            .thenReturn(mockS3ChecksumObject);
    retriever.getInventoryManifest();
}
 
Example #11
Source File: InventoryManifestRetrieverTest.java    From s3-inventory-usage-examples with Apache License 2.0 6 votes vote down vote up
@Test
public void getInventoryManifestSuccess() throws Exception {
    InventoryManifest expectedManifest = manifest();
    byte[] expectedManifestBytes = manifestBytes(expectedManifest);
    when(mockS3JsonObject.getObjectContent()).thenReturn(new S3ObjectInputStream(
            new ByteArrayInputStream(expectedManifestBytes), null));

    String expectedChecksum = "a6121a6a788be627a68d7e9def9f6968";
    byte[] expectedChecksumBytes = expectedChecksum.getBytes(StandardCharsets.UTF_8);
    when(mockS3ChecksumObject.getObjectContent()).thenReturn(new S3ObjectInputStream(
            new ByteArrayInputStream(expectedChecksumBytes), null));

    when(mockS3Client.getObject(getObjectRequestCaptor.capture()))
            .thenReturn(mockS3JsonObject)
            .thenReturn(mockS3ChecksumObject);
    InventoryManifest result = retriever.getInventoryManifest();
    assertThat(result, is(expectedManifest));

    List<GetObjectRequest> request = getObjectRequestCaptor.getAllValues();
    assertThat(request.get(0).getBucketName(), is("testBucketName"));
    assertThat(request.get(0).getKey(), is("testBucketKey/manifest.json"));
    assertThat(request.get(1).getBucketName(), is("testBucketName"));
    assertThat(request.get(1).getKey(), is("testBucketKey/manifest.checksum"));
}
 
Example #12
Source File: InventoryReportRetrieverTest.java    From s3-inventory-usage-examples with Apache License 2.0 6 votes vote down vote up
@Test
public void getInventReportSuccess() throws Exception {
    testLocator.setMD5checksum(testMD5);
    testManifest.setFileSchema("storageClass, size");
    reportRetriever = new InventoryReportRetriever(mockS3Client, testLocator, testManifest);

    String expectedInventoryReportString = "testString";
    byte[] expectedInventoryReportBytes = inventReportBytes(expectedInventoryReportString);
    when(mockS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(
            new ByteArrayInputStream(expectedInventoryReportBytes), null));
    when(mockS3Client.getObject(getObjectRequestCaptor.capture())).thenReturn(mockS3Object);

    String result = reportRetriever.getInventoryReportToString();
    assertThat(result, is(expectedInventoryReportString));

    GetObjectRequest request = getObjectRequestCaptor.getValue();
    assertThat(request.getBucketName(), is("testBucket"));
    assertThat(request.getKey(), is("testInventReportKey"));
}
 
Example #13
Source File: AwsS3BuildCacheServiceTest.java    From gradle-s3-build-cache with Apache License 2.0 6 votes vote down vote up
@Test
public void loadGetsObjectsAndReturnsTrueIfItExistsInS3() throws Exception {
  /** Setup **/
  buildCacheService = new AwsS3BuildCacheService(s3, "bucketName", null, true);
  doReturn(true).when(s3).doesObjectExist("bucketName", "abcdefghijkl123456789");
  S3Object s3Object = mock(S3Object.class);
  doReturn(s3Object).when(s3).getObject("bucketName", "abcdefghijkl123456789");
  S3ObjectInputStream s3ObjectInputStream = mock(S3ObjectInputStream.class);
  doReturn(s3ObjectInputStream).when(s3Object).getObjectContent();

  /** Run **/
  boolean result = buildCacheService.load(key, reader);

  /** Check **/
  assertTrue(result);
  verify(reader).readFrom(s3ObjectInputStream);
}
 
Example #14
Source File: MockAmazonS3.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public S3Object getObject(final GetObjectRequest request) throws AmazonClientException {
    assertThat(request.getBucketName(), equalTo(bucket));

    final String blobName = request.getKey();
    final byte[] content = blobs.get(blobName);
    if (content == null) {
        AmazonS3Exception exception = new AmazonS3Exception("[" + blobName + "] does not exist.");
        exception.setStatusCode(404);
        throw exception;
    }

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(content.length);

    S3Object s3Object = new S3Object();
    s3Object.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(content), null, false));
    s3Object.setKey(blobName);
    s3Object.setObjectMetadata(metadata);

    return s3Object;
}
 
Example #15
Source File: S3BucketStepsTests.java    From vividus with Apache License 2.0 6 votes vote down vote up
@Test
void fetchCsvFileTest() throws IOException
{
    String objectKey = S3_OBJECT_KEY + ".csv";
    byte[] csv = ResourceUtils.loadResourceAsByteArray(CSV_FILE_PATH);

    S3Object s3Object = mock(S3Object.class);
    S3ObjectInputStream s3ObjectInputStream = new S3ObjectInputStream(new ByteArrayInputStream(csv), null);
    when(s3Object.getObjectContent()).thenReturn(s3ObjectInputStream);
    when(amazonS3Client.getObject(S3_BUCKET_NAME, objectKey)).thenReturn(s3Object);

    Set<VariableScope> scopes = Set.of(VariableScope.SCENARIO);
    String variableName = "varName";
    steps.fetchCsvObject(objectKey, S3_BUCKET_NAME, scopes, variableName);
    verify(amazonS3Client).getObject(S3_BUCKET_NAME, objectKey);
    verify(bddVariableContext).putVariable(scopes, variableName, List.of(Map.of("id", "1")));
}
 
Example #16
Source File: S3Utils.java    From singleton with Eclipse Public License 2.0 6 votes vote down vote up
/**
 * convert the S3 Object to String
 */
public static String convertS3Obj2Str(S3Object s3Obj) throws IOException {
   S3ObjectInputStream s3is = s3Obj.getObjectContent();
   ByteArrayOutputStream fos = new ByteArrayOutputStream();
   byte[] read_buf = new byte[1024];
   int read_len = 0;
   try {
      while ((read_len = s3is.read(read_buf)) > 0) {
         fos.write(read_buf, 0, read_len);
      }
      return fos.toString(ConstantsUnicode.UTF8);
   } finally {
      s3is.close();
      fos.close();

   }
}
 
Example #17
Source File: FileReadingCollectorTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testCollectWithOneSocketTimeout() throws Throwable {
    S3ObjectInputStream inputStream = mock(S3ObjectInputStream.class);

    when(inputStream.read(any(byte[].class), anyInt(), anyInt()))
        .thenAnswer(new WriteBufferAnswer(new byte[]{102, 111, 111, 10}))  // first line: foo
        .thenThrow(new SocketTimeoutException())  // exception causes retry
        .thenAnswer(new WriteBufferAnswer(new byte[]{102, 111, 111, 10}))  // first line again, because of retry
        .thenAnswer(new WriteBufferAnswer(new byte[]{98, 97, 114, 10}))  // second line: bar
        .thenReturn(-1);

    TestingRowConsumer consumer = getObjects(Collections.singletonList("s3://fakebucket/foo"), null, inputStream, false);
    Bucket rows = consumer.getBucket();
    assertThat(rows.size(), is(2));
    assertThat(TestingHelpers.printedTable(rows), is("foo\nbar\n"));
}
 
Example #18
Source File: AmazonBucketClientImplTest.java    From molgenis with GNU Lesser General Public License v3.0 5 votes vote down vote up
@Test
void downloadFileExactTest() throws IOException {
  when(client.getObject(any())).thenReturn(s3Object);
  when(s3Object.getObjectContent())
      .thenReturn(
          new S3ObjectInputStream(
              new FileInputStream(ResourceUtils.getFile(getClass(), "/test_data_only.xlsx")),
              httpRequestBase));

  amazonBucketClient.downloadFile(client, fileStore, "ID", "bucket", "key", null, false, null);
  verify(fileStore).store(any(), eq("bucket_ID" + File.separatorChar + "key.xlsx"));
}
 
Example #19
Source File: FileReadingCollectorTest.java    From crate with Apache License 2.0 5 votes vote down vote up
private void getObjects(Collection<String> fileUris,
                        String compression,
                        final S3ObjectInputStream s3InputStream,
                        RowConsumer consumer,
                        boolean collectSourceUriFailure) throws Throwable {
    BatchIterator<Row> iterator = createBatchIterator(fileUris, compression, s3InputStream, collectSourceUriFailure);
    consumer.accept(iterator, null);
}
 
Example #20
Source File: FileReadingCollectorTest.java    From crate with Apache License 2.0 5 votes vote down vote up
private TestingRowConsumer getObjects(Collection<String> fileUris,
                                      String compression,
                                      S3ObjectInputStream s3InputStream,
                                      boolean collectSourceUriFailure) throws Throwable {
    TestingRowConsumer consumer = new TestingRowConsumer();
    getObjects(fileUris, compression, s3InputStream, consumer, collectSourceUriFailure);
    return consumer;
}
 
Example #21
Source File: S3Storage.java    From digdag with Apache License 2.0 5 votes vote down vote up
@Override
public StorageObject open(String key)
    throws StorageFileNotFoundException
{
    checkArgument(key != null, "key is null");

    String errorMessage = "opening file bucket " + bucket + " key " + key;
    GetObjectRequest req = new GetObjectRequest(bucket, key);

    S3Object obj = getWithRetry(errorMessage, () -> client.getObject(req));

    final long actualSize = obj.getObjectMetadata().getContentLength();

    // override close to call abort instead because close skips all remaining bytes so that
    // s3 client can reuse the TCP connection. but close of a fully opened file is occasionally
    // used to skip remaining work (e.g. finally block when exception is thrown). Unlike openRange,
    // performance impact could be significantly large.
    InputStream stream = overrideCloseToAbort(obj.getObjectContent());

    InputStream resumable = new ResumableInputStream(stream, (offset, closedCause) -> {
            try {
                S3ObjectInputStream raw = getWithRetry(errorMessage, () -> {
                        req.setRange(offset, actualSize - offset - 1);
                        return client.getObject(req);
                    })
                .getObjectContent();
                return overrideCloseToAbort(raw);
            }
            catch (StorageFileNotFoundException ex) {
                throw new IOException(ex);
            }
        });

    return new StorageObject(resumable, actualSize);
}
 
Example #22
Source File: S3Storage.java    From digdag with Apache License 2.0 5 votes vote down vote up
private InputStream overrideCloseToAbort(final S3ObjectInputStream raw)
{
    return new FilterInputStream(raw)
    {
        @Override
        public void close() throws IOException
        {
            raw.abort();
        }
    };
}
 
Example #23
Source File: FileReadingCollectorTest.java    From crate with Apache License 2.0 5 votes vote down vote up
private TestingRowConsumer getObjects(Collection<String> fileUris,
                                      String compression,
                                      boolean collectSourceUriFailure) throws Throwable {
    S3ObjectInputStream inputStream = mock(S3ObjectInputStream.class);
    when(inputStream.read(any(byte[].class), anyInt(), anyInt())).thenReturn(-1);
    return getObjects(fileUris, compression, inputStream, collectSourceUriFailure);
}
 
Example #24
Source File: S3RemoteFileSystem.java    From imhotep with Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws IOException {
    final long amtLeft;
    
    amtLeft = size - amtRead;
    if (amtLeft > 12000) {  // ~ 4 packets
        ((S3ObjectInputStream)in).abort();
    } else {
        in.close();
    }
}
 
Example #25
Source File: TestAmazonS3Executor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testCreateObject() throws Exception {
  AmazonS3ExecutorConfig config = getConfig();
  config.taskConfig.taskType = TaskType.CREATE_NEW_OBJECT;
  config.taskConfig.content = "${record:value('/content')}";

  AmazonS3Executor executor = new AmazonS3Executor(config);
  TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
    .build();
  runner.runInit();

  try {
    runner.runWrite(ImmutableList.of(getTestRecord()));

    //Make sure the prefix is empty
    ObjectListing objectListing = s3client.listObjects(BUCKET_NAME, objectName);
    Assert.assertEquals(1, objectListing.getObjectSummaries().size());

    S3Object object = s3client.getObject(BUCKET_NAME, objectName);
    S3ObjectInputStream objectContent = object.getObjectContent();

    List<String> stringList = IOUtils.readLines(objectContent);
    Assert.assertEquals(1, stringList.size());
    Assert.assertEquals("Secret", stringList.get(0));

    Assert.assertEquals(1, runner.getEventRecords().size());
    assertEvent(runner.getEventRecords().get(0), objectName);
  } finally {
    runner.runDestroy();
  }
}
 
Example #26
Source File: TestAmazonS3Executor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyObject() throws Exception {
  String newName = UUID.randomUUID().toString();

  AmazonS3ExecutorConfig config = getConfig();
  config.taskConfig.taskType = TaskType.COPY_OBJECT;
  config.taskConfig.copyTargetLocation = newName;

  AmazonS3Executor executor = new AmazonS3Executor(config);
  TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
    .build();
  runner.runInit();

  try {
    s3client.putObject(new PutObjectRequest(BUCKET_NAME, objectName, IOUtils.toInputStream("content"), new ObjectMetadata()));
    runner.runWrite(ImmutableList.of(getTestRecord()));

    S3Object object = s3client.getObject(BUCKET_NAME, newName);
    S3ObjectInputStream objectContent = object.getObjectContent();

    List<String> stringList = IOUtils.readLines(objectContent);
    Assert.assertEquals(1, stringList.size());
    Assert.assertEquals("content", stringList.get(0));

    Assert.assertTrue(s3client.doesObjectExist(BUCKET_NAME, objectName));

    Assert.assertEquals(1, runner.getEventRecords().size());
    assertEvent(runner.getEventRecords().get(0), newName);
  } finally {
    runner.runDestroy();
  }
}
 
Example #27
Source File: AmazonS3FileSystem.java    From iaf with Apache License 2.0 5 votes vote down vote up
@Override
public InputStream readFile(S3Object f) throws FileSystemException, IOException {
	try {
		final S3Object file = s3Client.getObject(bucketName, f.getKey());
		final S3ObjectInputStream is = file.getObjectContent();

		return is;
	} catch (AmazonServiceException e) {
		throw new FileSystemException(e);
	}
}
 
Example #28
Source File: TestAmazonS3Executor.java    From datacollector with Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyObjectDeleteOriginal() throws Exception {
  String newName = UUID.randomUUID().toString();

  AmazonS3ExecutorConfig config = getConfig();
  config.taskConfig.taskType = TaskType.COPY_OBJECT;
  config.taskConfig.dropAfterCopy = true;
  config.taskConfig.copyTargetLocation = newName;

  AmazonS3Executor executor = new AmazonS3Executor(config);
  TargetRunner runner = new TargetRunner.Builder(AmazonS3DExecutor.class, executor)
    .build();
  runner.runInit();

  try {
    s3client.putObject(new PutObjectRequest(BUCKET_NAME, objectName, IOUtils.toInputStream("dropAfterCopy"), new ObjectMetadata()));
    runner.runWrite(ImmutableList.of(getTestRecord()));

    S3Object object = s3client.getObject(BUCKET_NAME, newName);
    S3ObjectInputStream objectContent = object.getObjectContent();

    List<String> stringList = IOUtils.readLines(objectContent);
    Assert.assertEquals(1, stringList.size());
    Assert.assertEquals("dropAfterCopy", stringList.get(0));

    Assert.assertFalse(s3client.doesObjectExist(BUCKET_NAME, objectName));

    Assert.assertEquals(1, runner.getEventRecords().size());
    assertEvent(runner.getEventRecords().get(0), newName);
  } finally {
    runner.runDestroy();
  }
}
 
Example #29
Source File: S3Manager.java    From aws-cloudtrail-processing-library with Apache License 2.0 5 votes vote down vote up
/**
 * Downloads an AWS CloudTrail log from the specified source.
 *
 * @param ctLog The {@link CloudTrailLog} to download
 * @param source The {@link CloudTrailSource} to download the log from.
 * @return A byte array containing the log data.
 */
public byte[] downloadLog(CloudTrailLog ctLog, CloudTrailSource source) {
    boolean success = false;
    ProgressStatus downloadLogStatus = new ProgressStatus(ProgressState.downloadLog, new BasicProcessLogInfo(source, ctLog, success));
    final Object downloadSourceReportObject = progressReporter.reportStart(downloadLogStatus);

    byte[] s3ObjectBytes = null;

    // start to download CloudTrail log
    try {
        S3Object s3Object = this.getObject(ctLog.getS3Bucket(), ctLog.getS3ObjectKey());
        try (S3ObjectInputStream s3InputStream = s3Object.getObjectContent()){
            s3ObjectBytes = LibraryUtils.toByteArray(s3InputStream);
        }
        ctLog.setLogFileSize(s3Object.getObjectMetadata().getContentLength());
        success = true;
        logger.info("Downloaded log file " + ctLog.getS3ObjectKey() + " from " + ctLog.getS3Bucket());

    } catch (AmazonServiceException | IOException e) {
        String exceptionMessage = String.format("Fail to download log file %s/%s.", ctLog.getS3Bucket(), ctLog.getS3ObjectKey());
        LibraryUtils.handleException(exceptionHandler, downloadLogStatus, e, exceptionMessage);

    } finally {
        LibraryUtils.endToProcess(progressReporter, success, downloadLogStatus, downloadSourceReportObject);
    }

    return s3ObjectBytes;
}
 
Example #30
Source File: AwsPrivateKeyStoreTest.java    From athenz with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetApplicationSecret() {
    System.setProperty("athenz.aws.s3.region", "us-east-1");
    System.setProperty(ATHENZ_AWS_KMS_REGION, "us-east-1");
    String bucketName = "my_bucket";
    String keyName = "my_key";
    String expected = "my_value";

    AmazonS3 s3 = mock(AmazonS3.class);
    AWSKMS kms = mock(AWSKMS.class);
    S3Object s3Object = mock(S3Object.class);
    Mockito.when(s3.getObject(bucketName, keyName)).thenReturn(s3Object);
    InputStream is = new ByteArrayInputStream( expected.getBytes() );
    S3ObjectInputStream s3ObjectInputStream = new S3ObjectInputStream(is, null);
    Mockito.when(s3Object.getObjectContent()).thenReturn(s3ObjectInputStream);

    ByteBuffer buffer = ByteBuffer.wrap(expected.getBytes());
    DecryptResult decryptResult = mock(DecryptResult.class);
    Mockito.when(kms.decrypt(Mockito.any(DecryptRequest.class))).thenReturn(decryptResult);
    Mockito.when(decryptResult.getPlaintext()).thenReturn(buffer);

    System.setProperty("athenz.aws.store_kms_decrypt", "true");
    AwsPrivateKeyStore awsPrivateKeyStore = new AwsPrivateKeyStore();
    AwsPrivateKeyStore spyAWS = Mockito.spy(awsPrivateKeyStore);
    doReturn(s3).when(spyAWS).getS3();
    doReturn(kms).when(spyAWS).getKMS();
    String actual = spyAWS.getApplicationSecret(bucketName, keyName);
    Assert.assertEquals(actual, expected);
    System.clearProperty("athenz.aws.s3.region");
    System.clearProperty(ATHENZ_AWS_KMS_REGION);
}