org.jets3t.service.ServiceException Java Examples

The following examples show how to use org.jets3t.service.ServiceException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AmazonConnection.java    From Doradus with Apache License 2.0 6 votes vote down vote up
public List<ListItem> listAll(String path) {
    m_logger.debug("Start list all: " + path);
    try {
        List<ListItem> result = new ArrayList<>();
        String priorLastKey = null;
        while(true) {
            StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "/", CHUNK_SIZE, priorLastKey);
            m_logger.trace("ListObjects: {}", path);
            inc();
            StorageObject[] objects = chunk.getObjects();
            for(int i = 0; i < objects.length; i++) {
                String key = objects[i].getKey();
                if(key.endsWith("/")) key = key.substring(0, key.length() - 1);
                key = key.substring(path.length(), key.length());
                ListItem item = new ListItem(key, objects[i].getContentLength() != 0);
                result.add(item);
            }
            if(chunk.isListingComplete()) break;
            priorLastKey = chunk.getPriorLastKey();
        }
        return result;
   } catch (ServiceException e) {
        throw new RuntimeException(e);
    }
}
 
Example #2
Source File: S3HttpRequestRetryHandler.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Override
public boolean retryRequest(final IOException exception, final int executionCount, final HttpContext context) {
    if(super.retryRequest(exception, executionCount, context)) {
        final Object attribute = context.getAttribute(HttpCoreContext.HTTP_REQUEST);
        if(attribute instanceof HttpUriRequest) {
            final HttpUriRequest method = (HttpUriRequest) attribute;
            log.warn(String.format("Retrying request %s", method));
            try {
                // Build the authorization string for the method.
                authorizer.authorizeHttpRequest(method, context, null);
                return true;
            }
            catch(ServiceException e) {
                log.warn("Unable to generate updated authorization string for retried request", e);
            }
        }
    }
    return false;
}
 
Example #3
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void copy(String srcKey, String dstKey) throws IOException {
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
    }
    if (multipartEnabled) {
      S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
                                                   null, null, null);
      if (multipartCopyBlockSize > 0 &&
          object.getContentLength() > multipartCopyBlockSize) {
        copyLargeFile(object, dstKey);
        return;
      }
    }

    S3Object dstObject = new S3Object(dstKey);
    dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
        dstObject, false);
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 
Example #4
Source File: Jets3tNativeFileSystemStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void copy(String srcKey, String dstKey) throws IOException {
  try {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Copying srcKey: " + srcKey + "to dstKey: " + dstKey + "in bucket: " + bucket.getName());
    }
    if (multipartEnabled) {
      S3Object object = s3Service.getObjectDetails(bucket, srcKey, null,
                                                   null, null, null);
      if (multipartCopyBlockSize > 0 &&
          object.getContentLength() > multipartCopyBlockSize) {
        copyLargeFile(object, dstKey);
        return;
      }
    }

    S3Object dstObject = new S3Object(dstKey);
    dstObject.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.copyObject(bucket.getName(), srcKey, bucket.getName(),
        dstObject, false);
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 
Example #5
Source File: S3LoggingFeature.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Override
public void setConfiguration(final Path file, final LoggingConfiguration configuration) throws BackgroundException {
    // Logging target bucket
    final Path bucket = containerService.getContainer(file);
    try {
        final S3BucketLoggingStatus status = new S3BucketLoggingStatus(
                StringUtils.isNotBlank(configuration.getLoggingTarget()) ? configuration.getLoggingTarget() : bucket.getName(), null);
        if(configuration.isEnabled()) {
            status.setLogfilePrefix(PreferencesFactory.get().getProperty("s3.logging.prefix"));
        }
        session.getClient().setBucketLoggingStatus(bucket.getName(), status, true);
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Failure to write attributes of {0}", e, file);
    }
}
 
Example #6
Source File: S3LifecycleConfiguration.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Override
public void setConfiguration(final Path file, final LifecycleConfiguration configuration) throws BackgroundException {
    final Path container = containerService.getContainer(file);
    try {
        if(configuration.getTransition() != null || configuration.getExpiration() != null) {
            final LifecycleConfig config = new LifecycleConfig();
            // Unique identifier for the rule. The value cannot be longer than 255 characters. When you specify an empty prefix, the rule applies to all objects in the bucket
            final LifecycleConfig.Rule rule = config.newRule(
                    String.format("%s-%s", PreferencesFactory.get().getProperty("application.name"), new AlphanumericRandomStringService().random()), StringUtils.EMPTY, true);
            if(configuration.getTransition() != null) {
                rule.newTransition().setDays(configuration.getTransition());
            }
            if(configuration.getExpiration() != null) {
                rule.newExpiration().setDays(configuration.getExpiration());
            }
            session.getClient().setLifecycleConfig(container.getName(), config);
        }
        else {
            session.getClient().deleteLifecycleConfig(container.getName());
        }
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Failure to write attributes of {0}", e, container);
    }
}
 
Example #7
Source File: Jets3tNativeFileSystemStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * list objects
 * @param prefix prefix
 * @param delimiter delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      StorageObject object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (ServiceException e) {
    handleException(e, prefix);
    return null; // never returned - keep compiler happy
  }
}
 
Example #8
Source File: Jets3tNativeFileSystemStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key, long byteRangeStart)
        throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
        key, bucket.getName(), byteRangeStart);
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null;
  }
}
 
Example #9
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * list objects
 * @param prefix prefix
 * @param delimiter delimiter
 * @param maxListingLength max no. of entries
 * @param priorLastKey last key in any previous search
 * @return a list of matches
 * @throws IOException on any reported failure
 */

private PartialListing list(String prefix, String delimiter,
    int maxListingLength, String priorLastKey) throws IOException {
  try {
    if (!prefix.isEmpty() && !prefix.endsWith(PATH_DELIMITER)) {
      prefix += PATH_DELIMITER;
    }
    StorageObjectsChunk chunk = s3Service.listObjectsChunked(bucket.getName(),
        prefix, delimiter, maxListingLength, priorLastKey);
    
    FileMetadata[] fileMetadata =
      new FileMetadata[chunk.getObjects().length];
    for (int i = 0; i < fileMetadata.length; i++) {
      StorageObject object = chunk.getObjects()[i];
      fileMetadata[i] = new FileMetadata(object.getKey(),
          object.getContentLength(), object.getLastModifiedDate().getTime());
    }
    return new PartialListing(chunk.getPriorLastKey(), fileMetadata,
        chunk.getCommonPrefixes());
  } catch (ServiceException e) {
    handleException(e, prefix);
    return null; // never returned - keep compiler happy
  }
}
 
Example #10
Source File: S3MultipartWriteFeature.java    From cyberduck with GNU General Public License v3.0 6 votes vote down vote up
@Override
public HttpResponseOutputStream<VersionId> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService())
        .getDetails(file, status);
    // ID for the initiated multipart upload.
    final MultipartUpload multipart;
    try {
        multipart = session.getClient().multipartStartUpload(
            containerService.getContainer(file).getName(), object);
        if(log.isDebugEnabled()) {
            log.debug(String.format("Multipart upload started for %s with ID %s",
                multipart.getObjectKey(), multipart.getUploadId()));
        }
    }
    catch(ServiceException e) {
        throw new S3ExceptionMappingService().map("Upload {0} failed", e, file);
    }
    final MultipartOutputStream proxy = new MultipartOutputStream(multipart, file, status);
    return new HttpResponseOutputStream<VersionId>(new MemorySegementingOutputStream(proxy,
        preferences.getInteger("s3.upload.multipart.size"))) {
        @Override
        public VersionId getStatus() {
            return proxy.getVersionId();
        }
    };
}
 
Example #11
Source File: AmazonConnection.java    From Doradus with Apache License 2.0 6 votes vote down vote up
public void deleteAll(String path) {
    try {
        String priorLastKey = null;
        while(true) {
            StorageObjectsChunk chunk = m_s3service.listObjectsChunked(BUCKET, path, "?", CHUNK_SIZE, priorLastKey);
            m_logger.trace("ListObjects to delete: {}", path);
            inc();
            StorageObject[] objects = chunk.getObjects();
            if(objects.length == 0) break;
            String[] names = new String[objects.length];
            for(int i = 0; i < objects.length; i++) {
                names[i] = objects[i].getKey();
            }
            m_s3service.deleteMultipleObjects(BUCKET, names);
            m_logger.trace("DeleteObjects: {}", objects.length);
            // do not inc() because delete requests are not counted
            
            if(chunk.isListingComplete()) break;
            priorLastKey = chunk.getPriorLastKey();
        }
    } catch (ServiceException e) {
        throw new RuntimeException(e);
    }
}
 
Example #12
Source File: GrantAcl.java    From suro with Apache License 2.0 6 votes vote down vote up
public boolean grantAcl(S3Object object) throws ServiceException, InterruptedException {
    if(Strings.isNullOrEmpty(s3Acl)){
        return true;
    }

    for (int i = 0; i < s3AclRetries; ++i) {
        try {
            AccessControlList acl = s3Service.getObjectAcl(object.getBucketName(), object.getKey());
            for (String id : s3Acl.split(",")) {
                acl.grantPermission(new CanonicalGrantee(id), Permission.PERMISSION_READ);
            }
            s3Service.putObjectAcl(object.getBucketName(), object.getKey(), acl);
            return true;
        } catch (Exception e) {
            log.error("Exception while granting ACL: " + e.getMessage(), e);
            Thread.sleep(1000 * (i + 1));
        }
    }

    return false;

}
 
Example #13
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key, long byteRangeStart)
        throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {} with byteRangeStart: {}",
        key, bucket.getName(), byteRangeStart);
    S3Object object = s3Service.getObject(bucket, key, null, null, null,
                                          null, byteRangeStart, null);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null;
  }
}
 
Example #14
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
  StorageObject object = null;
  try {
    LOG.debug("Getting metadata for key: {} from bucket: {}",
        key, bucket.getName());
    object = s3Service.getObjectDetails(bucket.getName(), key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());

  } catch (ServiceException e) {
    try {
      // process
      handleException(e, key);
      return null;
    } catch (FileNotFoundException fnfe) {
      // and downgrade missing files
      return null;
    }
  } finally {
    if (object != null) {
      object.closeDataInputStream();
    }
  }
}
 
Example #15
Source File: Jets3tNativeFileSystemStore.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public FileMetadata retrieveMetadata(String key) throws IOException {
  StorageObject object = null;
  try {
    LOG.debug("Getting metadata for key: {} from bucket: {}",
        key, bucket.getName());
    object = s3Service.getObjectDetails(bucket.getName(), key);
    return new FileMetadata(key, object.getContentLength(),
        object.getLastModifiedDate().getTime());

  } catch (ServiceException e) {
    try {
      // process
      handleException(e, key);
      return null;
    } catch (FileNotFoundException fnfe) {
      // and downgrade missing files
      return null;
    }
  } finally {
    if (object != null) {
      object.closeDataInputStream();
    }
  }
}
 
Example #16
Source File: JetS3tStorage.java    From archistar-core with GNU General Public License v2.0 5 votes vote down vote up
@Override
public byte[] putBlob(String id, byte[] blob) throws DisconnectedException {

    if (s3service == null || s3bucket == null) {
        throw new DisconnectedException();
    }

    try {
        StorageObject obj = new S3Object(id, blob);
        s3service.putObject(s3bucket.getName(), obj);
        return blob;
    } catch (IOException | NoSuchAlgorithmException | ServiceException e) {
        throw new DisconnectedException();
    }
}
 
Example #17
Source File: AmazonFileStorageService.java    From computoser with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void delete(String path) throws IOException {
    path = stripLeadingSlash(path);
    try {
        service.deleteObject(bucketName, path);
    } catch (ServiceException ex) {
        throw new IOException(ex);
    }
}
 
Example #18
Source File: JetS3tStorage.java    From archistar-core with GNU General Public License v2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
public byte[] getBlob(String id) throws DisconnectedException {

    try {
        if (!fragmentExists(id)) {
            return new byte[0];
        }
        S3Object obj = s3service.getObject(s3bucket, id);
        return IOUtils.toByteArray(obj.getDataInputStream());
    } catch (ServiceException | IOException e) {
        throw new DisconnectedException();
    }
}
 
Example #19
Source File: TestS3FileSink.java    From suro with Apache License 2.0 5 votes vote down vote up
@Test
public void testAclFailure() throws IOException, ServiceException, InterruptedException {
    String testDir = tempDir.newFolder().getAbsolutePath();

    final String s3FileSink = "{\n" +
            "    \"type\": \"" + S3FileSink.TYPE + "\",\n" +
            "    \"localFileSink\": {\n" +
            "        \"type\": \"" + LocalFileSink.TYPE + "\",\n" +
            "        \"outputDir\": \"" + testDir + "\"\n" +
            "    },\n" +
            "    \"bucket\": \"s3bucket\"" +
            "}";

    Injector injector = getInjector();
    ObjectMapper mapper = injector.getInstance(ObjectMapper.class);

    S3FileSink sink = mapper.readValue(s3FileSink, new TypeReference<Sink>(){});
    GrantAcl grantAcl = mock(GrantAcl.class);
    when(grantAcl.grantAcl(any(S3Object.class))).thenReturn(false);
    sink.open();
    sink.grantAcl = grantAcl;

    for (Message m : new MessageSetReader(TestConnectionPool.createMessageSet(100000))) {
        sink.writeTo(new StringMessage(m));
    }
    sink.close();
    File[] files = getFiles(testDir);

    assertTrue(files.length > 0);
    int count = 0;
    while (sink.recvNotice() != null) {
        ++count;
    }
    assertEquals(count, 0);
}
 
Example #20
Source File: JetS3tStorage.java    From archistar-core with GNU General Public License v2.0 5 votes vote down vote up
@Override
public int disconnect() {
    s3bucket = null;
    try {
        s3service.shutdown();
    } catch (ServiceException e) {
        return -1;
    }

    s3service = null;
    return 0;
}
 
Example #21
Source File: AmazonFileStorageService.java    From computoser with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public void moveFile(String sourcePath, String targetPath)
        throws IOException {
    try {
        sourcePath = stripLeadingSlash(sourcePath);
        targetPath = stripLeadingSlash(targetPath);
        S3Object targetObject = new S3Object(targetPath);
        service.moveObject(bucketName, sourcePath, bucketName, targetObject, true);
    } catch (ServiceException ex) {
        throw new IOException(ex);
    }
}
 
Example #22
Source File: JetS3tStorage.java    From archistar-core with GNU General Public License v2.0 5 votes vote down vote up
private boolean fragmentExists(String id) {
    try {
        return s3service.isObjectInBucket(s3bucket.getName(), id);
    } catch (ServiceException e) {
        return false;
    }
}
 
Example #23
Source File: SpectraReadFeature.java    From cyberduck with GNU General Public License v3.0 5 votes vote down vote up
@Override
public InputStream read(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
    // Make sure file is available in cache
    final List<TransferStatus> chunks = bulk.query(Transfer.Type.download, file, status);
    // Sort chunks by offset
    chunks.sort(Comparator.comparingLong(TransferStatus::getOffset));
    final List<LazyInputStream> streams = new ArrayList<>();
    for(TransferStatus chunk : chunks) {
        final LazyInputStream in = new LazyInputStream(new LazyInputStream.OpenCallback() {
            @Override
            public InputStream open() throws IOException {
                try {
                    return session.getClient().getObjectImpl(
                        false,
                        containerService.getContainer(file).getName(),
                        containerService.getKey(file),
                        null,
                        null,
                        null,
                        null,
                        null,
                        null,
                        file.attributes().getVersionId(),
                        new HashMap<String, Object>(),
                        chunk.getParameters())
                        .getDataInputStream();
                }
                catch(ServiceException e) {
                    throw new IOException(e.getMessage(), e);
                }
            }
        });
        streams.add(in);
    }
    // Concatenate streams
    return new SequenceInputStream(Collections.enumeration(streams));
}
 
Example #24
Source File: Jets3tFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void handleServiceException(ServiceException e) throws IOException {
  if (e.getCause() instanceof IOException) {
    throw (IOException) e.getCause();
  }
  else {
    if(LOG.isDebugEnabled()) {
      LOG.debug("Got ServiceException with Error code: " + e.getErrorCode() + ";and Error message: " + e.getErrorMessage());
    }
  }
}
 
Example #25
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void copyLargeFile(S3Object srcObject, String dstKey) throws IOException {
  try {
    long partCount = srcObject.getContentLength() / multipartCopyBlockSize +
        (srcObject.getContentLength() % multipartCopyBlockSize > 0 ? 1 : 0);

    MultipartUpload multipartUpload = s3Service.multipartStartUpload
        (bucket.getName(), dstKey, srcObject.getMetadataMap());

    List<MultipartPart> listedParts = new ArrayList<MultipartPart>();
    for (int i = 0; i < partCount; i++) {
      long byteRangeStart = i * multipartCopyBlockSize;
      long byteLength;
      if (i < partCount - 1) {
        byteLength = multipartCopyBlockSize;
      } else {
        byteLength = srcObject.getContentLength() % multipartCopyBlockSize;
        if (byteLength == 0) {
          byteLength = multipartCopyBlockSize;
        }
      }

      MultipartPart copiedPart = s3Service.multipartUploadPartCopy
          (multipartUpload, i + 1, bucket.getName(), srcObject.getKey(),
           null, null, null, null, byteRangeStart,
           byteRangeStart + byteLength - 1, null);
      listedParts.add(copiedPart);
    }
    
    Collections.reverse(listedParts);
    s3Service.multipartCompleteUpload(multipartUpload, listedParts);
  } catch (ServiceException e) {
    handleException(e, srcObject.getKey());
  }
}
 
Example #26
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void rename(String srcKey, String dstKey) throws IOException {
  try {
    s3Service.renameObject(bucket.getName(), srcKey, new S3Object(dstKey));
  } catch (ServiceException e) {
    handleException(e, srcKey);
  }
}
 
Example #27
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void delete(String key) throws IOException {
  try {
    LOG.debug("Deleting key: {} from bucket: {}",
        key, bucket.getName());
    s3Service.deleteObject(bucket, key);
  } catch (ServiceException e) {
    handleException(e, key);
  }
}
 
Example #28
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * @param key
 * The key is the object name that is being retrieved from the S3 bucket
 * @return
 * This method returns null if the key is not found
 * @throws IOException
 */

@Override
public InputStream retrieve(String key) throws IOException {
  try {
    LOG.debug("Getting key: {} from bucket: {}",
        key, bucket.getName());
    S3Object object = s3Service.getObject(bucket.getName(), key);
    return object.getDataInputStream();
  } catch (ServiceException e) {
    handleException(e, key);
    return null; //return null if key not found
  }
}
 
Example #29
Source File: Jets3tNativeFileSystemStore.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void storeEmptyFile(String key) throws IOException {
  try {
    S3Object object = new S3Object(key);
    object.setDataInputStream(new ByteArrayInputStream(new byte[0]));
    object.setContentType("binary/octet-stream");
    object.setContentLength(0);
    object.setServerSideEncryptionAlgorithm(serverSideEncryptionAlgorithm);
    s3Service.putObject(bucket, object);
  } catch (ServiceException e) {
    handleException(e, key);
  }
}
 
Example #30
Source File: S3ExceptionMappingServiceTest.java    From cyberduck with GNU General Public License v3.0 5 votes vote down vote up
@Test
public void testHandshakeFailure() {
    final SSLHandshakeException f = new SSLHandshakeException("f");
    f.initCause(new CertificateException("c"));
    assertEquals(ConnectionCanceledException.class, new S3ExceptionMappingService().map(
        new ServiceException(f)).getClass());
}