Java Code Examples for com.amazonaws.services.s3.model.GetObjectRequest

The following examples show how to use com.amazonaws.services.s3.model.GetObjectRequest. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: cassandra-backup   Source File: S3Restorer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void downloadFile(final Path localPath, final RemoteObjectReference objectReference) throws Exception {
    final GetObjectRequest getObjectRequest = new GetObjectRequest(request.storageLocation.bucket, objectReference.canonicalPath);

    Files.createDirectories(localPath.getParent());

    final Optional<AmazonClientException> exception = ofNullable(transferManager.download(getObjectRequest,
                                                                                          localPath.toFile(),
                                                                                          new DownloadProgressListener(objectReference)).waitForException());

    if (exception.isPresent()) {
        if (exception.get() instanceof AmazonS3Exception && ((AmazonS3Exception) exception.get()).getStatusCode() == 404) {
            logger.error("Remote object reference {} does not exist.", objectReference);
        }

        throw exception.get();
    }
}
 
Example 2
Source Project: pacbot   Source File: AWSErrorManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(String datasource, List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+datasource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 
Example 3
Source Project: pacbot   Source File: ErrorManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Fetch error info.
 *
 * @param datasource the datasource
 * @param errorList the error list
 */
private void fetchErrorInfo(List<Map<String,String>> errorList){
	if(errorInfo==null){
		ObjectMapper objectMapper = new ObjectMapper();
    	List<Map<String, String>> inventoryErrors = new ArrayList<>();
    	AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(new CredentialProvider().getCredentials(s3Account,s3Role))).withRegion(s3Region).build();
    	try {
	    	S3Object inventoryErrorData = s3Client.getObject(new GetObjectRequest(bucketName,dataPath+"/"+dataSource+"-loaderror.data"));
	    	try (BufferedReader reader = new BufferedReader(new InputStreamReader(inventoryErrorData.getObjectContent()))) {
				inventoryErrors = objectMapper.readValue(reader.lines().collect(Collectors.joining("\n")),new TypeReference<List<Map<String, String>>>() {});
	        }
    	} catch (IOException e) {
    		LOGGER.error("Exception in collecting inventory error data",e);
            Map<String,String> errorMap = new HashMap<>();
            errorMap.put(ERROR, "Exception in collecting inventory error data");
            errorMap.put(ERROR_TYPE, WARN);
            errorMap.put(EXCEPTION, e.getMessage());
            errorList.add(errorMap);
		}
    	errorInfo = inventoryErrors.parallelStream().collect(Collectors.groupingBy(obj -> obj.get("type")));
	}
}
 
Example 4
/**
 * Get the original inventory report from S3, unzip it, and transfer it into a String format.
 * @return inventReport String
 * @throws IOException when getting object from S3 fails
 * or the checksum of the inventory report and the checksum specified in the manifest file not match
 */
public String getInventoryReportToString() throws IOException {
    String inventReportKey = locator.getKey();
    String bucketName = inventoryManifest.getSourceBucket();

    try (S3Object s3InventoryReport = s3Client.getObject(
            new GetObjectRequest(bucketName, inventReportKey))) {
        InputStream objectData = s3InventoryReport.getObjectContent();
        byte[] zippedData = IOUtils.toByteArray(objectData);
        String actualChecksum = DigestUtils.md5Hex(zippedData);
        String expectedChecksum = locator.getMD5checksum();
        if (!actualChecksum.equals(expectedChecksum)) {
            throw new ChecksumMismatchException (expectedChecksum, actualChecksum);
        }
        return IOUtils.toString(new GZIPInputStream(new ByteArrayInputStream(zippedData)));
    }
}
 
Example 5
/**
 * Check if the MD5s of manifest.json and manifest.checksum equal
 * if so, pull out the manifest file and map it into a POJO
 * @return inventoryManifestStorage InventoryManifest, which stores all the elements of the manifest.json file
 */
public InventoryManifest getInventoryManifest() throws Exception {
    // Get manifest.json and transfer it to String
    GetObjectRequest requestJson = new GetObjectRequest(bucketName, bucketKeyJson);
    S3Object jsonObject = s3Client.getObject(requestJson);
    String jsonFile = inputStreamToString(jsonObject.getObjectContent());
    jsonObject.close();

    // Get manifest.checksum and transfer it to String with no whitespace
    GetObjectRequest requestChecksum = new GetObjectRequest(bucketName, bucketKeyChecksum);
    S3Object checksumObject = s3Client.getObject(requestChecksum);
    String expectedChecksum = inputStreamToString(checksumObject.getObjectContent())
            .replaceAll("\\s","");
    checksumObject.close();

    // Compare manifest.json and manifest.checksum's MD5 value
    String actualChecksum = DigestUtils.md5Hex(jsonFile);
    if (!actualChecksum.equals(expectedChecksum)) {
        throw new ChecksumMismatchException (expectedChecksum, actualChecksum);
    }

    return mapper.readValue(jsonFile, InventoryManifest.class);
}
 
Example 6
@Test
public void getInventReportSuccess() throws Exception {
    testLocator.setMD5checksum(testMD5);
    testManifest.setFileSchema("storageClass, size");
    reportRetriever = new InventoryReportRetriever(mockS3Client, testLocator, testManifest);

    String expectedInventoryReportString = "testString";
    byte[] expectedInventoryReportBytes = inventReportBytes(expectedInventoryReportString);
    when(mockS3Object.getObjectContent()).thenReturn(new S3ObjectInputStream(
            new ByteArrayInputStream(expectedInventoryReportBytes), null));
    when(mockS3Client.getObject(getObjectRequestCaptor.capture())).thenReturn(mockS3Object);

    String result = reportRetriever.getInventoryReportToString();
    assertThat(result, is(expectedInventoryReportString));

    GetObjectRequest request = getObjectRequestCaptor.getValue();
    assertThat(request.getBucketName(), is("testBucket"));
    assertThat(request.getKey(), is("testInventReportKey"));
}
 
Example 7
Source Project: AWS-MIMIC-IIItoOMOP   Source File: Loader.java    License: Apache License 2.0 6 votes vote down vote up
public void load(Configuration configuration)
{
    String destination = "";
    
    setConfiguration(configuration);
    
    try {
        for(Mapping mapping : configuration.getMappings())
        {
            destination = mapping.getSqlFile().contains("/") ? mapping.getSqlFile().substring(mapping.getSqlFile().lastIndexOf("/") + 1, mapping.getSqlFile().length()).replace(".sql", "") : mapping.getSqlFile().replace(".sql", "");
            
            loadSourceFiles(mapping);
            
            df = loadDf(client.getObject(new GetObjectRequest(configuration.getSqlBucket(), mapping.getSqlFile())).getObjectContent());
            
            if(mapping.getOverflowColumns().equals(new ArrayList<String>())) { write(destination); }
            else { write(destination, mapping.getOverflowColumns()); }
            
            sns.publishSuccess("SparkBatchLoaderTableCompletion", destination);
        }
    } 
    catch (IOException ex) { sns.publishFailure("SparkBatchLoaderTableFailure", destination, ex.toString()); }
}
 
Example 8
private void setupS3(String fileName, String version, String propertyContent)
		throws UnsupportedEncodingException {
	final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", fileName);
	final GetObjectRequest request = new GetObjectRequest(s3ObjectId);

	final S3Object s3Object = new S3Object();
	s3Object.setObjectContent(new StringInputStream(propertyContent));

	if (version != null) {
		final ObjectMetadata metadata = new ObjectMetadata();
		metadata.setHeader("x-amz-version-id", version);
		s3Object.setObjectMetadata(metadata);
	}

	when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
			.thenReturn(s3Object);
}
 
Example 9
Source Project: genie   Source File: S3FileTransferImplTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test the getFile method for valid s3 path.
 *
 * @throws GenieException If there is any problem
 */
@Test
public void testGetFileMethodValidS3Path() throws GenieException {
    final ObjectMetadata objectMetadata = Mockito.mock(ObjectMetadata.class);
    Mockito.when(this.s3Client.getObject(Mockito.any(GetObjectRequest.class), Mockito.any(File.class)))
        .thenReturn(objectMetadata);
    final ArgumentCaptor<GetObjectRequest> argument = ArgumentCaptor.forClass(GetObjectRequest.class);

    s3FileTransfer.getFile(S3_PATH, LOCAL_PATH);
    Mockito.verify(this.s3Client).getObject(argument.capture(), Mockito.any());
    Assert.assertEquals(S3_BUCKET, argument.getValue().getBucketName());
    Assert.assertEquals(S3_KEY, argument.getValue().getKey());
    Mockito
        .verify(this.downloadTimer, Mockito.times(1))
        .record(Mockito.anyLong(), Mockito.eq(TimeUnit.NANOSECONDS));
    Mockito
        .verify(this.registry, Mockito.times(1))
        .timer(Mockito.eq(S3FileTransferImpl.DOWNLOAD_TIMER_NAME), this.tagsCaptor.capture());
    Assert.assertEquals(SUCCESS_TAGS, this.tagsCaptor.getValue());

}
 
Example 10
Source Project: konker-platform   Source File: AwsUploadRepository.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public InputStream downloadFile(String filePath) throws BusinessException {
    if (!Optional.ofNullable(filePath).isPresent()) {
        throw new BusinessException(Validations.INVALID_PATH.getCode());
    }
    client.getClient(credentials.getCredentials());
    try {
        if (filePath.contains("/")) {
            filePath = filePath.split("/")[filePath.split("/").length - 1];
        }

        S3Object object = s3Client.getObject(
                new GetObjectRequest(cdnConfig.getName(), filePath));
        return object.getObjectContent();
    } catch (AmazonServiceException e) {
        throw new BusinessException(Validations.INVALID_S3_BUCKET_CREDENTIALS.getCode());
    }
}
 
Example 11
Source Project: entrada   Source File: S3FileManagerImpl.java    License: GNU General Public License v3.0 6 votes vote down vote up
@Override
public Optional<InputStream> open(String location) {
  log.debug("Open S3 file: " + location);
  Optional<S3Details> details = S3Details.from(location);

  if (details.isPresent()) {
    try {
      return Optional
          .of(amazonS3
              .getObject(new GetObjectRequest(details.get().getBucket(), details.get().getKey()))
              .getObjectContent());
    } catch (Exception e) {
      log.error("Cannot open {}", location, e);
    }

  }

  return Optional.empty();
}
 
Example 12
public S3Object download(String bucketName, String key) throws AmazonServiceException {
  /*
   * Download an object - When you download an object, you get all of the
   * object's metadata and a stream from which to read the contents. It's
   * important to read the contents of the stream as quickly as possibly
   * since the data is streamed directly from Amazon S3 and your network
   * connection will remain open until you read all the data or close the
   * input stream.
   * 
   * GetObjectRequest also supports several other options, including
   * conditional downloading of objects based on modification times, ETags,
   * and selectively downloading a range of an object.
   */
  System.out.println("Downloading an object");
  S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, key));
  System.out.println("Content-Type: " + object.getObjectMetadata().getContentType());
  return object;
}
 
Example 13
Source Project: attic-apex-malhar   Source File: S3BlockReader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * S3 block read would be achieved through the AmazonS3 client. Following are the steps to achieve:
 * (1) Create the objectRequest from bucketName and filePath.
 * (2) Set the range to the above created objectRequest.
 * (3) Get the object portion through AmazonS3 client API.
 * (4) Get the object content from the above object portion.
 * @return the block entity
 * @throws IOException
 */
@Override
protected Entity readEntity() throws IOException
{
  entity.clear();
  GetObjectRequest rangeObjectRequest = new GetObjectRequest(
      bucketName, filePath);
  rangeObjectRequest.setRange(offset, blockMetadata.getLength() - 1);
  S3Object objectPortion = s3Client.getObject(rangeObjectRequest);
  S3ObjectInputStream wrappedStream = objectPortion.getObjectContent();
  byte[] record = ByteStreams.toByteArray(wrappedStream);
  entity.setUsedBytes(record.length);
  entity.setRecord(record);
  wrappedStream.close();
  return entity;
}
 
Example 14
Source Project: ReCiter   Source File: DynamoDbS3Operations.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This function retrieves large object from S3
 * @param bucketName
 * @param keyName
 * @param objectClass
 * @return
 */
public <T> Object retrieveLargeItem(String bucketName, String keyName, Class<T> objectClass) {
	try {
		S3Object s3Object = s3.getObject(new GetObjectRequest(bucketName.toLowerCase(), keyName));
		String objectContent = IOUtils.toString(s3Object.getObjectContent(), StandardCharsets.UTF_8);
		if(objectClass == ReCiterFeature.class) {
			ReCiterFeature reCiterFeature = OBJECT_MAPPER.readValue(objectContent, ReCiterFeature.class);
			return reCiterFeature;
		}
		
	} catch (IOException | AmazonServiceException e) {
		log.error(e.getMessage());
	}
	return null;
	
}
 
Example 15
Source Project: datacollector   Source File: AmazonS3Util.java    License: Apache License 2.0 6 votes vote down vote up
static S3Object getObject(
    AmazonS3 s3Client,
    String bucket,
    String objectKey,
    boolean useSSE,
    CredentialValue customerKey,
    CredentialValue customerKeyMd5
) throws StageException {
  GetObjectRequest getObjectRequest = new GetObjectRequest(bucket, objectKey);
  if (useSSE) {
    SSECustomerKey sseCustomerKey = new SSECustomerKey(customerKey.get());
    sseCustomerKey.setMd5(customerKeyMd5.get());
    getObjectRequest.setSSECustomerKey(sseCustomerKey);
  }
  return s3Client.getObject(getObjectRequest);
}
 
Example 16
Source Project: aws-ant-tasks   Source File: UploadFileSetToS3TaskTests.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExecuteSingleFile() throws FileNotFoundException,
        IOException {
    UploadFileSetToS3Task task = new UploadFileSetToS3Task();
    task.setProject(new Project());
    FileSet fileset = new FileSet();
    fileset.setDir(testFile1.getParentFile());
    fileset.setFile(testFile1);
    task.addFileset(fileset);
    task.setBucketName(BUCKET_NAME);
    task.setKeyPrefix(KEY_PREFIX);
    task.execute();
    resFile1 = File.createTempFile(RES_FILE_1, TESTFILE_SUFFIX);
    client.getObject(new GetObjectRequest(BUCKET_NAME, KEY_PREFIX
            + fileName1), resFile1);
    assertTrue(FileUtils.contentEquals(testFile1, resFile1));
}
 
Example 17
Source Project: genie   Source File: S3FileTransferImplTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test the getFile method for valid s3 path.
 *
 * @throws GenieException If there is any problem
 */
@Test(expected = GenieServerException.class)
public void testGetFileMethodFailureToFetch() throws GenieException {
    Mockito.when(this.s3Client.getObject(Mockito.any(GetObjectRequest.class), Mockito.any(File.class)))
        .thenThrow(new AmazonS3Exception("something"));
    final ArgumentCaptor<GetObjectRequest> argument = ArgumentCaptor.forClass(GetObjectRequest.class);

    try {
        this.s3FileTransfer.getFile(S3_PATH, LOCAL_PATH);
    } finally {
        Mockito.verify(this.s3Client).getObject(argument.capture(), Mockito.any());
        Assert.assertEquals(S3_BUCKET, argument.getValue().getBucketName());
        Assert.assertEquals(S3_KEY, argument.getValue().getKey());
        Mockito
            .verify(this.downloadTimer, Mockito.times(1))
            .record(Mockito.anyLong(), Mockito.eq(TimeUnit.NANOSECONDS));
        Mockito
            .verify(this.registry, Mockito.times(1))
            .timer(Mockito.eq(S3FileTransferImpl.DOWNLOAD_TIMER_NAME), this.tagsCaptor.capture());
        Assert.assertEquals(
            MetricsUtils.newFailureTagsSetForException(new GenieServerException("blah")),
            this.tagsCaptor.getValue()
        );
    }
}
 
Example 18
Source Project: render   Source File: S3URLConnection.java    License: GNU General Public License v2.0 6 votes vote down vote up
private synchronized void connectToS3() throws IOException {
    if (! connected) {
        try {
            String s3key;
            try {
                s3key = java.net.URLDecoder.decode(s3uri.getKey(), "UTF-8");
            } catch (final UnsupportedEncodingException e) {
                LOG.warn("failed to decode key, using raw key instead", e);
                // TODO: Better error handling with badly encoded URLs?
                s3key = s3uri.getKey();
            }
            s3object = s3Client.getObject(new GetObjectRequest(s3uri.getBucket(), s3key));
            connected = true;
        } catch (final AmazonServiceException ase) {
            throw new IOException("Amazon S3 service failure for error type " + ase.getErrorType(), ase);
        } catch (final AmazonClientException ace) {
            throw new IOException("Amazon S3 client failure", ace);
        }
    }
}
 
Example 19
Source Project: presto   Source File: S3TableConfigClient.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Connect to S3 directory to look for new or updated table definitions and then
 * update the map.
 */
private void updateTablesFromS3()
{
    long now = System.currentTimeMillis();

    AmazonS3Client s3client = clientManager.getS3Client();

    for (S3ObjectSummary summary : getObjectSummaries()) {
        if (!descriptors.containsKey(summary.getKey()) || summary.getLastModified().getTime() >= lastCheck) {
            // New or updated file, so we must read from AWS
            if (summary.getKey().endsWith("/")) {
                continue;
            }

            log.info("Getting : %s - %s", summary.getBucketName(), summary.getKey());
            S3Object object = s3client.getObject(new GetObjectRequest(summary.getBucketName(), summary.getKey()));

            try (BufferedReader reader = new BufferedReader(new InputStreamReader(object.getObjectContent(), UTF_8))) {
                KinesisStreamDescription table = streamDescriptionCodec.fromJson(CharStreams.toString(reader));
                descriptors.put(summary.getKey(), table);
                log.info("Put table description into the map from %s", summary.getKey());
            }
            catch (IOException iox) {
                log.error("Problem reading input stream from object.", iox);
                throwIfUnchecked(iox);
                throw new RuntimeException(iox);
            }
        }
    }

    log.info("Completed updating table definitions from S3.");
    lastCheck = now;
}
 
Example 20
Source Project: presto   Source File: MockAmazonS3.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public S3Object getObject(GetObjectRequest getObjectRequest)
{
    if (getObjectHttpCode != HTTP_OK) {
        AmazonS3Exception exception = new AmazonS3Exception("Failing getObject call with " + getObjectHttpCode);
        exception.setStatusCode(getObjectHttpCode);
        throw exception;
    }
    return null;
}
 
Example 21
Source Project: bidder   Source File: AwsCommander.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Load the file or s3 object.
 * @param parts String[]. An array of tokens.
 * @return String. The message returned from the load command.
 * @throws Exception on I/O errirs.
 */
String load(String[] parts) throws Exception {
	String otype = null;
	String symbolName = null;
	String name;
	String type = parts[1]; // file or S3
	//for (int i=0;i<parts.length;i++) {
	//	System.out.println("Part[" + i + "] = " + parts[i]);
	//}
	if (type.equalsIgnoreCase("S3")) {
		
		otype = parts[2]; // bloom, cache, cuckoo.
		name = parts[4];
		symbolName = parts[3]; // name of the object
		if (!symbolName.startsWith("$"))
			symbolName = "$" + symbolName;
	} else
		name = parts[2]; // file name

	if (type.equals("file")) {
		return Configuration.getInstance().readData(parts[2]);
	} 
	
	S3Object object = Configuration.s3.getObject(new GetObjectRequest(Configuration.s3_bucket,name));
	long size = Configuration.s3.getObjectMetadata(Configuration.s3_bucket, name).getContentLength();
	
	return Configuration.getInstance().readData(otype,symbolName,object, size);
}
 
Example 22
Source Project: cassandra-backup   Source File: S3Restorer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String downloadFileToString(final Path localPath, final RemoteObjectReference objectReference) throws Exception {
    final GetObjectRequest getObjectRequest = new GetObjectRequest(request.storageLocation.bucket, objectReference.canonicalPath);
    try (final InputStream is = transferManager.getAmazonS3Client().getObject(getObjectRequest).getObjectContent(); final InputStreamReader isr = new InputStreamReader(is)) {
        return CharStreams.toString(isr);
    }
}
 
Example 23
Source Project: jpms-module-names   Source File: Bucket.java    License: MIT License 5 votes vote down vote up
private Path toPath(String key) throws IOException {
  // Expecting key format: "modulescanner-report-2018_08_18_00_58_06.csv"
  var year = Integer.valueOf(key.substring(21, 25));
  var zipFileSystem = zips.get(year);
  if (zipFileSystem != null) {
    var zip = zipFileSystem.getPath(key);
    if (Files.exists(zip)) {
      LOG.log(DEBUG, "Extracting {0} from {1}...", key, zipFileSystem);
      return zip;
    }
  }
  var csv = cache.resolve(key);
  if (Files.notExists(csv)) {
    Files.createDirectories(csv.getParent());
    LOG.log(INFO, "Downloading {0} from remote {1}...", key, bucketName);
    Files.createDirectories(cache);
    try (var object = s3.getObject(new GetObjectRequest(bucketName, key))) {
      var length = object.getObjectMetadata().getContentLength();
      LOG.log(INFO, "Loading {0} bytes to {1}...", length, csv);
      try (var stream = object.getObjectContent().getDelegateStream()) {
        Files.copy(stream, csv);
      }
    }
    LOG.log(DEBUG, "Loaded {0} bytes to {1}", Files.size(csv), csv);
  }
  return csv;
}
 
Example 24
Source Project: hadoop   Source File: S3AInputStream.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void reopen(long pos) throws IOException {

    if (wrappedStream != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Aborting old stream to open at pos " + pos);
      }
      wrappedStream.abort();
    }

    if (pos < 0) {
      throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK
          +" " + pos);
    }

    if (contentLength > 0 && pos > contentLength-1) {
      throw new EOFException(
          FSExceptionMessages.CANNOT_SEEK_PAST_EOF
          + " " + pos);
    }

    LOG.debug("Actually opening file " + key + " at pos " + pos);

    GetObjectRequest request = new GetObjectRequest(bucket, key);
    request.setRange(pos, contentLength-1);

    wrappedStream = client.getObject(request).getObjectContent();

    if (wrappedStream == null) {
      throw new IOException("Null IO stream");
    }

    this.pos = pos;
  }
 
Example 25
Source Project: AWS-MIMIC-IIItoOMOP   Source File: FileProcessor.java    License: Apache License 2.0 5 votes vote down vote up
public void readFiles() throws IOException, SQLException 
{
    String query = "";
    S3Object s3object = client.getObject(new GetObjectRequest(bucket, System.getenv("prefix") +"/" + table + ".sql"));
    
    statement = connection.createStatement();
        
    query = IOUtils.toString(s3object.getObjectContent(), "UTF-8");
    query = query.replace("${redshift_arn}", System.getenv("redshift_arn"));
    query = query.replace("${bucket}", System.getenv("bucket"));
    
    statement.execute(query);
}
 
Example 26
Source Project: flink-crawler   Source File: SeedUrlSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    super.open(parameters);

    StreamingRuntimeContext context = (StreamingRuntimeContext) getRuntimeContext();
    int parallelism = context.getNumberOfParallelSubtasks();
    if (parallelism != 1) {
        throw new IllegalStateException("SeedUrlSource only supports a parallelism of 1");
    }

    if (_terminator == null) {
        throw new IllegalStateException("Crawl terminator must be set for the seed URL source");
    }
    
    LOGGER.info("Opening seed URL source");

    // Open the terminator, so that it knows when we really started running.
    _terminator.open();
    
    _urlIndex = 0;
    
    if (useS3File()) {
        AmazonS3 s3Client = S3Utils.makeS3Client();
        S3Object object = s3Client
                .getObject(new GetObjectRequest(_seedUrlsS3Bucket, _seedUrlsS3Path));
        _s3FileStream = object.getObjectContent();
    }
}
 
Example 27
Source Project: big-c   Source File: S3AInputStream.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void reopen(long pos) throws IOException {

    if (wrappedStream != null) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Aborting old stream to open at pos " + pos);
      }
      wrappedStream.abort();
    }

    if (pos < 0) {
      throw new EOFException(FSExceptionMessages.NEGATIVE_SEEK
          +" " + pos);
    }

    if (contentLength > 0 && pos > contentLength-1) {
      throw new EOFException(
          FSExceptionMessages.CANNOT_SEEK_PAST_EOF
          + " " + pos);
    }

    LOG.debug("Actually opening file " + key + " at pos " + pos);

    GetObjectRequest request = new GetObjectRequest(bucket, key);
    request.setRange(pos, contentLength-1);

    wrappedStream = client.getObject(request).getObjectContent();

    if (wrappedStream == null) {
      throw new IOException("Null IO stream");
    }

    this.pos = pos;
  }
 
Example 28
Source Project: emodb   Source File: RestartingS3InputStream.java    License: Apache License 2.0 5 votes vote down vote up
public RestartingS3InputStream(AmazonS3 s3, String bucket, String key, @Nullable Range<Long> range) {
    _s3 = s3;
    _bucket = bucket;
    _key = key;

    S3Object s3Object;

    // Get the object synchronously so any immediate S3 errors, such as file not found, are thrown inline.
    if (range == null) {
        s3Object = _s3.getObject(new GetObjectRequest(_bucket, _key));
        _pos = 0;
        _length = s3Object.getObjectMetadata().getContentLength();
    } else {
        long start, end;

        if (range.hasLowerBound()) {
            start = range.lowerEndpoint() + (range.lowerBoundType() == BoundType.CLOSED ? 0 : 1);
        } else {
            start = 0;
        }

        if (range.hasUpperBound()) {
            end = range.upperEndpoint() - (range.upperBoundType() == BoundType.CLOSED ? 0 : 1);
        } else {
            end = Long.MAX_VALUE;
        }

        s3Object = _s3.getObject(new GetObjectRequest(_bucket, _key).withRange(start, end));

        _pos = start;
        // The S3 metadata's content length is the length of the data actually being returned by S3.
        // Since we effectively skipped the first "start" bytes we need to add them back to the total length
        // of data being read to make future calculations using _pos and _length consistent.
        _length = start + s3Object.getObjectMetadata().getContentLength();
    }

    _in = s3Object.getObjectContent();
}
 
Example 29
Source Project: emodb   Source File: RestartingS3InputStream.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Re-opens the input stream, starting at the first unread byte.
 */
private void reopenS3InputStream()
        throws IOException {
    // First attempt to close the existing input stream
    try {
        closeS3InputStream();
    } catch (IOException ignore) {
        // Ignore this exception; we're re-opening because there was in issue with the existing stream
        // in the first place.
    }

    InputStream remainingIn = null;
    int attempt = 0;
    while (remainingIn == null) {
        try {
            S3Object s3Object = _s3.getObject(
                    new GetObjectRequest(_bucket, _key)
                            .withRange(_pos, _length - 1));  // Range is inclusive, hence length-1

            remainingIn = s3Object.getObjectContent();
        } catch (AmazonClientException e) {
            // Allow up to 3 retries
            attempt += 1;
            if (!e.isRetryable() || attempt == 4) {
                throw e;
            }
            // Back-off on each retry
            try {
                Thread.sleep(200 * attempt);
            } catch (InterruptedException interrupt) {
                throw Throwables.propagate(interrupt);
            }
        }
    }

    _in = remainingIn;
}
 
Example 30
Source Project: emodb   Source File: StashReaderTest.java    License: Apache License 2.0 5 votes vote down vote up
private Matcher<GetObjectRequest> getsObject(final String bucket, final String key) {
    return new BaseMatcher<GetObjectRequest>() {
        @Override
        public boolean matches(Object o) {
            GetObjectRequest request = (GetObjectRequest) o;
            return request != null && request.getBucketName().equals(bucket) && request.getKey().equals(key);
        }

        @Override
        public void describeTo(Description description) {
            description.appendText("gets object s3://").appendText(bucket).appendText("/").appendText(key);
        }
    };
}
 
Example 31
Source Project: StubbornJava   Source File: S3Client.java    License: MIT License 5 votes vote down vote up
public String getString(String bucket, String key) {
    GetObjectRequest request = new GetObjectRequest(bucket, key);
    S3Object response = client.getObject(request);
    try (S3ObjectInputStream is = response.getObjectContent()) {
        return CharStreams.toString(new InputStreamReader(is, Charsets.UTF_8));
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}
 
Example 32
Source Project: imhotep   Source File: S3RemoteFileSystem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void copyFileInto(String fullPath, File localFile) throws IOException {
    final String relativePath = mounter.getMountRelativePath(fullPath, mountPoint);
    final String s3path = getS3path(relativePath);
    final ObjectMetadata metadata;
    
    try {
        metadata = client.getObject(new GetObjectRequest(s3bucket, s3path), localFile);
    } catch(AmazonS3Exception e) {
        throw new IOException(e);
    }
}
 
Example 33
Source Project: athenz   Source File: S3ChangeLogStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSignedDomainNotFound() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenReturn(null);
    
    assertNull(store.getSignedDomain(store.awsS3Client, "iaas"));
}
 
Example 34
Source Project: athenz   Source File: S3ChangeLogStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSignedDomainClientException() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    
    when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonClientException("failed client operation"));
    assertNull(store.getSignedDomain(store.awsS3Client, "iaas"));
}
 
Example 35
Source Project: athenz   Source File: S3ChangeLogStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetSignedDomainServiceException() {
    MockS3ChangeLogStore store = new MockS3ChangeLogStore(null);
    
    when(store.awsS3Client.getObject(any(GetObjectRequest.class))).thenThrow(new AmazonServiceException("failed server operation"));
    assertNull(store.getSignedDomain(store.awsS3Client, "iaas"));
}
 
Example 36
Source Project: oodt   Source File: TestS3DataTransferer.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRetrieveProduct() throws DataTransferException, IOException {
	dataTransferer.retrieveProduct(product, stagingDir);

	ArgumentCaptor<GetObjectRequest> argument = ArgumentCaptor.forClass(GetObjectRequest.class);
	verify(s3Client).getObject(argument.capture());

	GetObjectRequest request = argument.getValue();
	assertThat(request.getBucketName(), is(S3_BUCKET_NAME));
	assertThat(request.getKey(), is(EXPECTED_DATA_STORE_REF));
}
 
Example 37
Source Project: herd   Source File: S3DaoImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Retrieves an S3 object.
 *
 * @param s3Client the S3 client
 * @param bucketName the S3 bucket name
 * @param key the S3 object key
 * @param errorOnNoSuchKey true to throw an error when the object key is not found, otherwise return null
 *
 * @return the S3 object
 * @throws ObjectNotFoundException when specified bucket or key does not exist or access to bucket or key is denied
 */
private S3Object getS3Object(AmazonS3Client s3Client, String bucketName, String key, boolean errorOnNoSuchKey)
{
    try
    {
        GetObjectRequest getObjectRequest = new GetObjectRequest(bucketName, key);
        return s3Operations.getS3Object(getObjectRequest, s3Client);
    }
    catch (AmazonServiceException amazonServiceException)
    {
        String errorCode = amazonServiceException.getErrorCode();

        if (S3Operations.ERROR_CODE_ACCESS_DENIED.equals(errorCode))
        {
            throw new ObjectNotFoundException(
                "Application does not have access to the specified S3 object at bucket '" + bucketName + "' and key '" + key + "'.",
                amazonServiceException);
        }
        else if (S3Operations.ERROR_CODE_NO_SUCH_BUCKET.equals(errorCode))
        {
            throw new ObjectNotFoundException("Specified S3 bucket '" + bucketName + "' does not exist.", amazonServiceException);
        }
        else if (S3Operations.ERROR_CODE_NO_SUCH_KEY.equals(errorCode))
        {
            if (errorOnNoSuchKey)
            {
                throw new ObjectNotFoundException("Specified S3 object key '" + key + "' does not exist.", amazonServiceException);
            }
            else
            {
                return null;
            }
        }
        else
        {
            throw amazonServiceException;
        }
    }
}
 
Example 38
Source Project: herd   Source File: MockS3OperationsImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public S3Object getS3Object(GetObjectRequest getObjectRequest, AmazonS3 s3)
{
    MockS3Object mockS3Object = getMockS3Object(getObjectRequest.getBucketName(), getObjectRequest.getKey());

    S3Object s3Object = new S3Object();
    s3Object.setBucketName(getObjectRequest.getBucketName());
    s3Object.setKey(getObjectRequest.getKey());
    s3Object.setObjectContent(new ByteArrayInputStream(mockS3Object.getData()));
    s3Object.setObjectMetadata(mockS3Object.getObjectMetadata());

    return s3Object;
}
 
Example 39
/**
 *
 * @param Id
 * @param Ver
 * @return
 * @throws PDException
 */
protected InputStream Retrieve(String Id, String Ver, Record Rec) throws PDException
{
try {
S3Object object = s3.getObject(new GetObjectRequest(BucketName, GenKey(Id, Ver)));
return(object.getObjectContent());
} catch (Exception ex)
    {
    PDException.GenPDException("Error_retrieving_content", ex.getLocalizedMessage());
    }
return(null);
}
 
Example 40
Source Project: zeppelin   Source File: S3NotebookRepo.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Note get(String noteId, String notePath, AuthenticationInfo subject) throws IOException {
  S3Object s3object;
  try {
    s3object = s3client.getObject(new GetObjectRequest(bucketName,
        rootFolder + "/" + buildNoteFileName(noteId, notePath)));
  }
  catch (AmazonClientException ace) {
    throw new IOException("Fail to get note: " + notePath + " from S3", ace);
  }
  try (InputStream ins = s3object.getObjectContent()) {
    String json = IOUtils.toString(ins, conf.getString(ConfVars.ZEPPELIN_ENCODING));
    return Note.fromJson(json);
  }
}