Java Code Examples for com.amazonaws.services.s3.model.S3ObjectSummary#getETag()

The following examples show how to use com.amazonaws.services.s3.model.S3ObjectSummary#getETag() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StorageObjectSummary.java    From snowflake-jdbc with Apache License 2.0 5 votes vote down vote up
/**
 * Contructs a StorageObjectSummary object from the S3 equivalent S3ObjectSummary
 *
 * @param objSummary the AWS S3 ObjectSummary object to copy from
 * @return the ObjectSummary object created
 */
public static StorageObjectSummary createFromS3ObjectSummary(S3ObjectSummary objSummary)
{

  return new StorageObjectSummary(
      objSummary.getBucketName(),
      objSummary.getKey(),
      // S3 ETag is not always MD5, but since this code path is only
      // used in skip duplicate files in PUT command, It's not
      // critical to guarantee that it's MD5
      objSummary.getETag(),
      objSummary.getSize()
  );
}
 
Example 2
Source File: S3Spooler.java    From datacollector with Apache License 2.0 4 votes vote down vote up
private void findAndQueueObjects(
    AmazonS3Source amazonS3Source, BatchContext batchContext
) throws AmazonClientException {
  S3Offset s3offset;
  if (lastElementAddedToQueue != null) {
    s3offset = lastElementAddedToQueue;
  } else {
    s3offset = amazonS3Source.getLatestOffset();
  }

  List<S3ObjectSummary> s3ObjectSummaries;
  if (!AWSUtil.containsWildcard(s3ConfigBean.s3FileConfig.prefixPattern)) {
    // No wildcard in the prefixPattern - don't need to scan the bucket
    s3ObjectSummaries = AmazonS3Util.getObjectNoWildcard(s3Client, s3ConfigBean, s3offset, s3ConfigBean.s3FileConfig.prefixPattern);
  } else {
    ObjectOrdering objectOrdering = s3ConfigBean.s3FileConfig.objectOrdering;
    switch (objectOrdering) {
      case TIMESTAMP:
        s3ObjectSummaries = AmazonS3Util.listObjectsChronologically(s3Client,
            s3ConfigBean,
            pathMatcher,
            s3offset,
            objectQueue.remainingCapacity()
        );
        break;
      case LEXICOGRAPHICAL:
        s3ObjectSummaries = AmazonS3Util.listObjectsLexicographically(s3Client,
            s3ConfigBean,
            pathMatcher,
            s3offset,
            objectQueue.remainingCapacity()
        );
        break;
      default:
        throw new IllegalArgumentException("Unknown ordering: " + objectOrdering.getLabel());
    }
  }
  for (S3ObjectSummary objectSummary : s3ObjectSummaries) {
    addObjectToQueue(objectSummary);
  }
  spoolQueueMeter.mark(objectQueue.size());
  LOG.debug("Found '{}' files", objectQueue.size());
  if (s3ObjectSummaries.isEmpty()) {
    // Before sending the event we will check that all the threads have finished with their objects, if yes, we
    // send the event as normal, if not we will skip and try to send it again if the queue is still empty when the
    // next thread tries to fill the queue. If the event is sent we will set the newDataAfterEventSent to false to
    // indicate that we should not send new events until we get more new data
    if (newDataAfterEventSent) {
      newDataAfterEventSent = !amazonS3Source.sendNoMoreDataEvent(batchContext);
    }
  } else {
    // If it is the last element save it to keep track of the last element added to the queue
    S3ObjectSummary s3ObjectSummary = s3ObjectSummaries.get(s3ObjectSummaries.size() - 1);
    lastElementAddedToQueue = new S3Offset(s3ObjectSummary.getKey(),
        S3Constants.MINUS_ONE,
        s3ObjectSummary.getETag(),
        String.valueOf(s3ObjectSummary.getLastModified().getTime())
    );

    //  If we previously sent a no-more-data event and we have new objects now, let's reset the event to be able to
    //  send it again.
    if (!newDataAfterEventSent) {
      amazonS3Source.restartNoMoreDataEvent();
      newDataAfterEventSent = true;
    }
  }
}
 
Example 3
Source File: BucketClass.java    From cloudExplorer with GNU General Public License v3.0 4 votes vote down vote up
String getObjectInfo(String key, String access_key,
        String secret_key, String bucket,
        String endpoint, String process
) {
    AWSCredentials credentials = new BasicAWSCredentials(access_key, secret_key);
    AmazonS3 s3Client = new AmazonS3Client(credentials,
            new ClientConfiguration());
    if (endpoint.contains("amazonaws.com")) {
        String aws_endpoint = s3Client.getBucketLocation(new GetBucketLocationRequest(bucket));
        if (aws_endpoint.contains("US")) {
            s3Client.setEndpoint("https://s3.amazonaws.com");
        } else if (aws_endpoint.contains("us-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("eu-west")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("ap-")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else if (aws_endpoint.contains("sa-east-1")) {
            s3Client.setEndpoint("https://s3-" + aws_endpoint + ".amazonaws.com");
        } else {
            s3Client.setEndpoint("https://s3." + aws_endpoint + ".amazonaws.com");
        }
    } else {
        s3Client.setS3ClientOptions(S3ClientOptions.builder().setPathStyleAccess(true).build());
        s3Client.setEndpoint(endpoint);
    }
    objectlist = null;

    try {
        ObjectListing current = s3Client.listObjects((bucket));

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket);
        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(listObjectsRequest);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                if (process.contains("checkmd5")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = objectSummary.getETag();
                        break;
                    }
                }
                if (process.contains("objectsize")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getSize());
                        break;
                    }
                }

                if (process.contains("objectdate")) {
                    if (objectSummary.getKey().contains(key)) {
                        objectlist = String.valueOf(objectSummary.getLastModified());
                        break;
                    }

                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());

    } catch (Exception listBucket) {
        //  mainFrame.jTextArea1.append("\n" + listBucket.getMessage());
    }

    return objectlist;
}