org.jets3t.service.ServiceException Java Examples

public List<ListItem> listAll(String path) {
    m_logger.debug("Start list all: " + path);
    try {
        List<ListItem> result = new ArrayList<>();
        String priorLastKey = null;
        while(true) {
            StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey);
            m_logger.trace("ListObjects: {}", path);
            inc();
            StorageObject[] objects = chunk.getObjects();
            for(int i = 0; i < objects.length; i++) {
                String key = objects[i].getKey();
                if(key.endsWith("/")) key = key.substring(0, key.length() - 1);
                key = key.substring(path.length(), key.length());
                ListItem item = new ListItem(key, objects[i].getContentLength() != 0);
                result.add(item);
            }
            if(chunk.isListingComplete()) break;
            priorLastKey = chunk.getPriorLastKey();
        }
        return result;
   } catch (ServiceException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
    if(super.retryRequest(exception, executionCount, context)) {
        final Object attribute = context.getAttribute(HttpCoreContext.HTTP_REQUEST);
        if(attribute instanceof HttpUriRequest) {
            final HttpUriRequest method = (HttpUriRequest) attribute;
            log.warn(String.format("Retrying request %s", method));
            try {
                // Build the authorization string for the method.
                authorizer.authorizeHttpRequest(method, context, null);
                return true;
            }
            catch(ServiceException e) {
                log.warn("Unable to generate updated authorization string for retried request", e);
            }
        }
    }
    return false;
}
 

@Override
public void copy(String srcKey, String dstKey) throws IOException {
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
    }
    if (multipartEnabled) {
      S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
                                                   null, null, null);
      if (multipartCopyBlockSize > 0 &&
          object.getContentLength() > multipartCopyBlockSize) {
        copyLargeFile(object, dstKey);
        return;
      }
    }

    S3Object dstObject = new S3Object(dstKey);
    dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
        dstObject, false);
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 

@Override
public void copy(String srcKey, String dstKey) throws IOException {
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
    }
    if (multipartEnabled) {
      S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
                                                   null, null, null);
      if (multipartCopyBlockSize > 0 &&
          object.getContentLength() > multipartCopyBlockSize) {
        copyLargeFile(object, dstKey);
        return;
      }
    }

    S3Object dstObject = new S3Object(dstKey);
    dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
        dstObject, false);
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 

@Override
public void setConfiguration(final Path file, final LoggingConfiguration configuration) throws BackgroundException {
    // Logging target bucket
    final Path bucket = containerService.getContainer(file);
    try {
        final S3BucketLoggingStatus status = new S3BucketLoggingStatus(
                StringUtils.isNotBlank(configuration.getLoggingTarget()) ? configuration.getLoggingTarget() : bucket.getName(), null);
        if(configuration.isEnabled()) {
            status.setLogfilePrefix(PreferencesFactory.get().getProperty("s3.logging.prefix"));
        }
        session.getClient().setBucketLoggingStatus(bucket.getName(), status, true);
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Failure to write attributes of {0}", e, file);
    }
}
 

@Override
public void setConfiguration(final Path file, final LifecycleConfiguration configuration) throws BackgroundException {
    final Path container = containerService.getContainer(file);
    try {
        if(configuration.getTransition() != null || configuration.getExpiration() != null) {
            final LifecycleConfig config = new LifecycleConfig();
            // Unique identifier for the rule. The value cannot be longer than 255 characters. When you specify an empty prefix, the rule applies to all objects in the bucket
            final LifecycleConfig.Rule rule = config.newRule(
                    String.format("%s-%s", PreferencesFactory.get().getProperty("application.name"), new AlphanumericRandomStringService().random()), StringUtils.EMPTY, true);
            if(configuration.getTransition() != null) {
                rule.newTransition().setDays(configuration.getTransition());
            }
            if(configuration.getExpiration() != null) {
                rule.newExpiration().setDays(configuration.getExpiration());
            }
            session.getClient().setLifecycleConfig(container.getName(), config);
        }
        else {
            session.getClient().deleteLifecycleConfig(container.getName());
        }
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Failure to write attributes of {0}", e, container);
    }
}
 

/**
 * list objects
 * @param prefix prefix
 * @param delimiter delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      StorageObject object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (ServiceException e) {
    handleException(e, prefix);
    return null; // never returned - keep compiler happy
  }
}
 

/**
 *
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key, long byteRangeStart)
        throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
        key, bucket.getName(), byteRangeStart);
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null;
  }
}
 

/**
 * list objects
 * @param prefix prefix
 * @param delimiter delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      StorageObject object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (ServiceException e) {
    handleException(e, prefix);
    return null; // never returned - keep compiler happy
  }
}
 

@Override
public HttpResponseOutputStream<VersionId> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService())
        .getDetails(file, status);
    // ID for the initiated multipart upload.
    final MultipartUpload multipart;
    try {
        multipart = session.getClient().multipartStartUpload(
            containerService.getContainer(file).getName(), object);
        if(log.isDebugEnabled()) {
            log.debug(String.format("Multipart upload started for %s with ID %s",
                multipart.getObjectKey(), multipart.getUploadId()));
        }
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Upload {0} failed", e, file);
    }
    final MultipartOutputStream proxy = new MultipartOutputStream(multipart, file, status);
    return new HttpResponseOutputStream<VersionId>(new MemorySegementingOutputStream(proxy,
        preferences.getInteger("s3.upload.multipart.size"))) {
        @Override
        public VersionId getStatus() {
            return proxy.getVersionId();
        }
    };
}
 

public void deleteAll(String path) {
    try {
        String priorLastKey = null;
        while(true) {
            StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "?", CHUNK_SIZE, priorLastKey);
            m_logger.trace("ListObjects to delete: {}", path);
            inc();
            StorageObject[] objects = chunk.getObjects();
            if(objects.length == 0) break;
            String[] names = new String[objects.length];
            for(int i = 0; i < objects.length; i++) {
                names[i] = objects[i].getKey();
            }
            m_s3service.deleteMultipleObjects(BUCKET, names);
            m_logger.trace("DeleteObjects: {}", objects.length);
            // do not inc() because delete requests are not counted
            
            if(chunk.isListingComplete()) break;
            priorLastKey = chunk.getPriorLastKey();
        }
    } catch (ServiceException e) {
        throw new RuntimeException(e);
    }
}
 

public boolean grantAcl(S3Object object) throws ServiceException, InterruptedException {
    if(Strings.isNullOrEmpty(s3Acl)){
        return true;
    }

    for (int i = 0; i < s3AclRetries; ++i) {
        try {
            AccessControlList acl = s3Service.getObjectAcl(object.getBucketName(), object.getKey());
            for (String id : s3Acl.split(",")) {
                acl.grantPermission(new CanonicalGrantee(id), Permission.PERMISSION_READ);
            }
            s3Service.putObjectAcl(object.getBucketName(), object.getKey(), acl);
            return true;
        } catch (Exception e) {
            log.error("Exception while granting ACL: " + e.getMessage(), e);
            Thread.sleep(1000 * (i + 1));
        }
    }

    return false;

}
 

/**
 *
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key, long byteRangeStart)
        throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
        key, bucket.getName(), byteRangeStart);
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null;
  }
}
 

@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
  StorageObject object = null;
  try {
    LOG.debug("Getting metadata for key: {} from bucket: {}",
        key, bucket.getName());
    object = s3Service.getObjectDetails(bucket.getName(), key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());

  } catch (ServiceException e) {
    try {
      // process
      handleException(e, key);
      return null;
    } catch (FileNotFoundException fnfe) {
      // and downgrade missing files
      return null;
    }
  } finally {
    if (object != null) {
      object.closeDataInputStream();
    }
  }
}
 

@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
  StorageObject object = null;
  try {
    LOG.debug("Getting metadata for key: {} from bucket: {}",
        key, bucket.getName());
    object = s3Service.getObjectDetails(bucket.getName(), key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());

  } catch (ServiceException e) {
    try {
      // process
      handleException(e, key);
      return null;
    } catch (FileNotFoundException fnfe) {
      // and downgrade missing files
      return null;
    }
  } finally {
    if (object != null) {
      object.closeDataInputStream();
    }
  }
}
 

@Override
public byte[] putBlob(String id, byte[] blob) throws DisconnectedException {

    if (s3service == null || s3bucket == null) {
        throw new DisconnectedException();
    }

    try {
        StorageObject obj = new S3Object(id, blob);
        s3service.putObject(s3bucket.getName(), obj);
        return blob;
    } catch (IOException | NoSuchAlgorithmException | ServiceException e) {
        throw new DisconnectedException();
    }
}
 

@Override
public void delete(String path) throws IOException {
    path = stripLeadingSlash(path);
    try {
        service.deleteObject(bucketName, path);
    } catch (ServiceException ex) {
        throw new IOException(ex);
    }
}
 

@SuppressWarnings("deprecation")
@Override
public byte[] getBlob(String id) throws DisconnectedException {

    try {
        if (!fragmentExists(id)) {
            return new byte[0];
        }
        S3Object obj = s3service.getObject(s3bucket, id);
        return IOUtils.toByteArray(obj.getDataInputStream());
    } catch (ServiceException | IOException e) {
        throw new DisconnectedException();
    }
}
 

@Test
public void testAclFailure() throws IOException, ServiceException, InterruptedException {
    String testDir = tempDir.newFolder().getAbsolutePath();

    final String s3FileSink = "{\n" +
            "    \"type\": \"" + S3FileSink.TYPE + "\",\n" +
            "    \"localFileSink\": {\n" +
            "        \"type\": \"" + LocalFileSink.TYPE + "\",\n" +
            "        \"outputDir\": \"" + testDir + "\"\n" +
            "    },\n" +
            "    \"bucket\": \"s3bucket\"" +
            "}";

    Injector injector = getInjector();
    ObjectMapper mapper = injector.getInstance(ObjectMapper.class);

    S3FileSink sink = mapper.readValue(s3FileSink, new TypeReference<Sink>(){});
    GrantAcl grantAcl = mock(GrantAcl.class);
    when(grantAcl.grantAcl(any(S3Object.class))).thenReturn(false);
    sink.open();
    sink.grantAcl = grantAcl;

    for (Message m : new MessageSetReader(TestConnectionPool.createMessageSet(100000))) {
        sink.writeTo(new StringMessage(m));
    }
    sink.close();
    File[] files = getFiles(testDir);

    assertTrue(files.length > 0);
    int count = 0;
    while (sink.recvNotice() != null) {
        ++count;
    }
    assertEquals(count, 0);
}
 

@Override
public int disconnect() {
    s3bucket = null;
    try {
        s3service.shutdown();
    } catch (ServiceException e) {
        return -1;
    }

    s3service = null;
    return 0;
}
 

@Override
public void moveFile(String sourcePath, String targetPath)
        throws IOException {
    try {
        sourcePath = stripLeadingSlash(sourcePath);
        targetPath = stripLeadingSlash(targetPath);
        S3Object targetObject = new S3Object(targetPath);
        service.moveObject(bucketName, sourcePath, bucketName, targetObject, true);
    } catch (ServiceException ex) {
        throw new IOException(ex);
    }
}
 

private boolean fragmentExists(String id) {
    try {
        return s3service.isObjectInBucket(s3bucket.getName(), id);
    } catch (ServiceException e) {
        return false;
    }
}
 

@Override
public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    // Make sure file is available in cache
    final List<TransferStatus> chunks = bulk.query(Transfer.Type.download, file, status);
    // Sort chunks by offset
    chunks.sort(Comparator.comparingLong(TransferStatus::getOffset));
    final List<LazyInputStream> streams = new ArrayList<>();
    for(TransferStatus chunk : chunks) {
        final LazyInputStream in = new LazyInputStream(new LazyInputStream.OpenCallback() {
            @Override
            public InputStream open() throws IOException {
                try {
                    return session.getClient().getObjectImpl(
                        false,
                        containerService.getContainer(file).getName(),
                        containerService.getKey(file),
                        null,
                        null,
                        null,
                        null,
                        null,
                        null,
                        file.attributes().getVersionId(),
                        new HashMap<String, Object>(),
                        chunk.getParameters())
                        .getDataInputStream();
                }
                catch(ServiceException e) {
                    throw new IOException(e.getMessage(), e);
                }
            }
        });
        streams.add(in);
    }
    // Concatenate streams
    return new SequenceInputStream(Collections.enumeration(streams));
}
 

private void handleServiceException(ServiceException e) throws IOException {
  if (e.getCause() instanceof IOException) {
    throw (IOException) e.getCause();
  }
  else {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage());
    }
  }
}
 

public void copyLargeFile(S3Object srcObject, String dstKey) throws IOException {
  try {
    long partCount = srcObject.getContentLength() / multipartCopyBlockSize +
        (srcObject.getContentLength() % multipartCopyBlockSize > 0 ? 1 : 0);

    MultipartUpload multipartUpload = s3Service.multipartStartUpload
        (bucket.getName(), dstKey, srcObject.getMetadataMap());

    List<MultipartPart> listedParts = new ArrayList<MultipartPart>();
    for (int i = 0; i < partCount; i++) {
      long byteRangeStart = i * multipartCopyBlockSize;
      long byteLength;
      if (i < partCount - 1) {
        byteLength = multipartCopyBlockSize;
      } else {
        byteLength = srcObject.getContentLength() % multipartCopyBlockSize;
        if (byteLength == 0) {
          byteLength = multipartCopyBlockSize;
        }
      }

      MultipartPart copiedPart = s3Service.multipartUploadPartCopy
          (multipartUpload, i + 1, bucket.getName(), srcObject.getKey(),
           null, null, null, null, byteRangeStart,
           byteRangeStart + byteLength - 1, null);
      listedParts.add(copiedPart);
    }
    
    Collections.reverse(listedParts);
    s3Service.multipartCompleteUpload(multipartUpload, listedParts);
  } catch (ServiceException e) {
    handleException(e, srcObject.getKey());
  }
}
 

public void rename(String srcKey, String dstKey) throws IOException {
  try {
    s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey));
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 

@Override
public void delete(String key) throws IOException {
  try {
    LOG.debug("Deleting key: {} from bucket: {}",
        key, bucket.getName());
    s3Service.deleteObject(bucket, key);
  } catch (ServiceException e) {
    handleException(e, key);
  }
}
 

/**
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key) throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {}",
        key, bucket.getName());
    S3Object object = s3Service.getObject(bucket.getName(), key);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null; //return null if key not found
  }
}
 

@Override
public void storeEmptyFile(String key) throws IOException {
  try {
    S3Object object = new S3Object(key);
    object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
    object.setContentType("binary/octet-stream");
    object.setContentLength(0);
    object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.putObject(bucket, object);
  } catch (ServiceException e) {
    handleException(e, key);
  }
}
 

@Test
public void testHandshakeFailure() {
    final SSLHandshakeException f = new SSLHandshakeException("f");
    f.initCause(new CertificateException("c"));
    assertEquals(ConnectionCanceledException.class, new S3ExceptionMappingService().map(
        new ServiceException(f)).getClass());
}