org.jclouds.blobstore.domain.StorageMetadata Java Examples

/**
 * {@inheritDoc}
 */
@Override
public InputStream getInputStream(String id, String root, String filePath) throws IOException {
    ContainerAndName can = getContainerAndName(id, root, filePath);
    Blob blob = getBlobStore().getBlob(can.container, can.name);
    if (blob == null){
        throw new IOException("No object found for " + id);
    }

    StorageMetadata metadata = blob.getMetadata();
    Long size = metadata.getSize();

    if (size != null && size.longValue() > maxBlobStreamSize) {
        return streamFromTempFile(blob, size);
    } else {
        // SAK-30325: why can't we just send the stream straight back: blob.getPayload().openStream() ?
        // Good question, but it doesn't work properly unless the stream is fully copied and re-streamed....
        return new ByteArrayInputStream(FileCopyUtils.copyToByteArray(blob.getPayload().openStream()));
    }
}
 

public void testCreateListDestroySimpleDirInContainer() throws IOException {
    context.getBlobStore().createContainerInLocation(null, testContainerName);
    context.getBlobStore().createDirectory(testContainerName, "my-dir-1");
    
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list(testContainerName);
    assertHasItemNamed(ps, "my-dir-1");
    
    Blob b = context.getBlobStore().blobBuilder("my-blob-1").payload(Streams.newInputStreamWithContents("hello world")).build();
    context.getBlobStore().putBlob(testContainerName+"/"+"my-dir-1", b);
    
    ps = context.getBlobStore().list(testContainerName, ListContainerOptions.Builder.inDirectory("my-dir-1"));
    assertHasItemNamed(ps, "my-dir-1/my-blob-1");

    // both these syntaxes work:
    Blob b2 = context.getBlobStore().getBlob(testContainerName+"/"+"my-dir-1", "my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b2.getPayload().openStream()), "hello world");

    Blob b3 = context.getBlobStore().getBlob(testContainerName, "my-dir-1"+"/"+"my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b3.getPayload().openStream()), "hello world");

    context.getBlobStore().deleteContainer(testContainerName);
}
 

/**
 * {@inheritDoc}
 */
@Override
public InputStream getInputStream(String id, String root, String filePath) throws IOException {
    ContainerAndName can = getContainerAndName(id, root, filePath);
    Blob blob = getBlobStore().getBlob(can.container, can.name);
    if (blob == null){
        throw new IOException("No object found for " + id);
    }

    StorageMetadata metadata = blob.getMetadata();
    Long size = metadata.getSize();

    if (size != null && size.longValue() > maxBlobStreamSize) {
        return streamFromTempFile(blob, size);
    } else {
        // SAK-30325: why can't we just send the stream straight back: blob.getPayload().openStream() ?
        // Good question, but it doesn't work properly unless the stream is fully copied and re-streamed....
        return new ByteArrayInputStream(FileCopyUtils.copyToByteArray(blob.getPayload().openStream()));
    }
}
 

@Override
public boolean deleteApp( App app )
{
    log.info( "Deleting app " + app.getName() );

    // Delete all files related to app
    for ( StorageMetadata resource : blobStore.list( config.container, prefix( app.getFolderName() ).recursive() ) )
    {
        log.info( "Deleting app file: " + resource.getName() );

        blobStore.removeBlob( config.container, resource.getName() );
    }

    reservedNamespaces.remove( app.getActivities().getDhis().getNamespace(), app );

    log.info( "Deleted app " + app.getName() );

    return true;
}
 

@Test
public void testCreateBlobGetBlob() throws Exception {
    String blobName = createRandomBlobName();
    Blob blob = makeBlob(nullBlobStore, blobName);
    nullBlobStore.putBlob(containerName, blob);

    blob = nullBlobStore.getBlob(containerName, blobName);
    validateBlobMetadata(blob.getMetadata());

    // content differs, only compare length
    try (InputStream actual = blob.getPayload().openStream();
            InputStream expected = BYTE_SOURCE.openStream()) {
        long actualLength = ByteStreams.copy(actual,
                ByteStreams.nullOutputStream());
        long expectedLength = ByteStreams.copy(expected,
                ByteStreams.nullOutputStream());
        assertThat(actualLength).isEqualTo(expectedLength);
    }

    PageSet<? extends StorageMetadata> pageSet = nullBlobStore.list(
            containerName);
    assertThat(pageSet).hasSize(1);
    StorageMetadata sm = pageSet.iterator().next();
    assertThat(sm.getName()).isEqualTo(blobName);
    assertThat(sm.getSize()).isEqualTo(0);
}
 

private boolean filterByModificationTime(StorageMetadata blobMetadata, Date modificationTime) {
    Map<String, String> userMetadata = blobMetadata.getUserMetadata();
    // Clean up any blobStore entries that don't have any metadata as we can't check their creation date
    if (CollectionUtils.isEmpty(userMetadata)) {
        return true;
    }
    String longString = userMetadata.get(Constants.FILE_ENTRY_MODIFIED.toLowerCase());
    try {
        long dateLong = Long.parseLong(longString);
        Date date = new Date(dateLong);
        return date.before(modificationTime);
    } catch (NumberFormatException e) {
        // Clean up any blobStore entries that have invalid timestamp
        return true;
    }
}
 

private Set<String> listAllBlobs() {
    Set<String> blobNames = new HashSet<String>();
    ListContainerOptions options = new ListContainerOptions();
    while (true) {
        PageSet<? extends StorageMetadata> set = blobStoreRead.list(
                containerName, options);
        for (StorageMetadata sm : set) {
            blobNames.add(sm.getName());
        }
        String marker = set.getNextMarker();
        if (marker == null) {
            break;
        }
        options = options.afterMarker(marker);
    }
    return blobNames;
}
 

public synchronized boolean hasNewFiles() {
    // see if there are any files since lastMarker.
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);
    
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);
    
    log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker);
    boolean emptiness = getBlobsWithinRange(pages).isEmpty();

    if(emptiness) {
        log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME));
    } else {
        log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME));
    }

    return !emptiness;
}
 

private NavigableMap<Long,String> getBlobsWithinRange(PageSet<? extends StorageMetadata> pages) {
    // TreeMap used because of sorted property
    TreeMap<Long, String> tsToBlobName = new TreeMap<Long, String>();
    for (StorageMetadata blobMeta : pages) {
        String fileName = blobMeta.getName(); // 20140226_1393442533000.json.gz
        String dateAndTs = fileName.split("\\.", 2)[0].trim(); // 20140226_1393442533000
        String tsCreated = dateAndTs.split("_")[1].trim(); // 1393442533000
        long ts = Long.parseLong(tsCreated);
        tsToBlobName.put(ts, fileName);
    }
    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = tsToBlobName.subMap(START_TIME - 60000*15, true, STOP_TIME + 60000*30, true);
    if(mapWithinRange.isEmpty()) {
        lastMarker = tsToBlobName.lastEntry().getValue().trim();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(tsToBlobName.lastEntry().getValue().trim());
        }
    }
    return mapWithinRange;
}
 

public void testCreateListDestroyContainer() throws IOException {
    context.getBlobStore().createContainerInLocation(null, testContainerName);
    context.getBlobStore().list(testContainerName);
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
    assertHasItemNamed(ps, testContainerName);
    
    Blob b = context.getBlobStore().blobBuilder("my-blob-1").payload(Streams.newInputStreamWithContents("hello world")).build();
    context.getBlobStore().putBlob(testContainerName, b);
    
    Blob b2 = context.getBlobStore().getBlob(testContainerName, "my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b2.getPayload().openStream()), "hello world");
    
    context.getBlobStore().deleteContainer(testContainerName);
}
 

@Override
public PageSet<? extends StorageMetadata> list(String container) {
    ImmutableSet.Builder<StorageMetadata> builder = ImmutableSet.builder();
    PageSet<? extends StorageMetadata> pageSet = super.list(container);
    for (StorageMetadata sm : pageSet) {
        MutableStorageMetadata msm = new MutableStorageMetadataImpl(sm);
        msm.setSize(0L);
        builder.add(msm);
    }
    return new PageSetImpl<>(builder.build(), pageSet.getNextMarker());
}
 

public synchronized void downloadNewFiles(File downloadDir) {
    log.info("Downloading new files since {}", lastMarker);
    
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);

    // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);

    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages);

    //Download only for keys within that range
    for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) {
        log.info("Downloading file: " + blobMeta.getValue());
        downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue()));
        lastMarker = blobMeta.getValue();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(blobMeta.getValue());
        }
    }
    log.info("Updated the last marker value as " + lastMarker);
}
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

@Override
protected void after() {
    logger.debug("S3 proxy is stopping");
    try {
        s3Proxy.stop();
        BlobStore blobStore = blobStoreContext.getBlobStore();
        for (StorageMetadata metadata : blobStore.list()) {
            blobStore.deleteContainer(metadata.getName());
        }
        blobStoreContext.close();
    } catch (Exception e) {
        throw new RuntimeException("Unable to stop S3 proxy", e);
    }
    FileUtils.deleteQuietly(blobStoreLocation);
    logger.debug("S3 proxy has stopped");
}
 

@Override
protected void doCall(BlobStore blobStore, String indent) throws Exception {
    for (String containerName : names) {
        Set<? extends StorageMetadata> contents = blobStore.list(containerName);
        stdout.println(indent+"Container "+containerName+" {");
        for (StorageMetadata content : contents) {
            stdout.println(indent+"\t"+content);
        }
        stdout.println(indent+"}");
    }
}
 

@Override
protected void doCall(BlobStore blobstore, String indent) throws Exception {
    Set<? extends StorageMetadata> containers = blobstore.list();
    stdout.println(indent+"Containers {");
    for (StorageMetadata container : containers) {
        stdout.println(indent+"\t"+container);
    }
    stdout.println(indent+"}");
}
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

public static void main(String[] args) {
        LocalManagementContextForTests mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault()); 
        JcloudsLocation location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
            
        String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null");
        String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null");
        String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
        String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);

        BlobStoreContext context = ContextBuilder.newBuilder(provider)
                .credentials(identity, credential)
                .endpoint(endpoint)
                .buildView(BlobStoreContext.class);
        
        PageSet<? extends StorageMetadata> containers = context.getBlobStore().list();
        for (StorageMetadata container: containers) {
            if (container.getName().matches("brooklyn.*-test.*")
                // to kill all containers here
//                || container.getName().matches(".*")
                ) {
                log.info("killing - "+container.getName());
                context.getBlobStore().deleteContainer(container.getName());
            } else {
                log.info("KEEPING - "+container.getName());
            }
        }
        context.close();
    }
 

private StorageMetadata toStorageMetadata(OSS oss, String container, OSSObjectSummary ossObjectSummary) {
    ObjectMetadata metadata = oss.getObjectMetadata(container, ossObjectSummary.getKey());
    URI url = getPresignedUriForObject(oss, ossObjectSummary);
    return new StorageMetadataImpl(StorageType.BLOB, ossObjectSummary.getKey(), ossObjectSummary.getKey(), defaultLocation.get(),
                                   url, ossObjectSummary.getETag(), ossObjectSummary.getLastModified(),
                                   ossObjectSummary.getLastModified(), metadata.getUserMetadata(), ossObjectSummary.getSize(),
                                   Tier.STANDARD);
}
 

@Override
public List<FileEntry> getFileEntriesWithoutContent(List<FileEntry> fileEntries) {
    Set<String> existingFiles = blobStore.list(container)
                                         .stream()
                                         .map(StorageMetadata::getName)
                                         .collect(Collectors.toSet());

    return fileEntries.stream()
                      .filter(fileEntry -> {
                          String id = fileEntry.getId();
                          return !existingFiles.contains(id);
                      })
                      .collect(Collectors.toList());
}
 

private int removeBlobsByFilter(Predicate<? super StorageMetadata> filter) {
    Set<String> entries = getEntryNames(filter);
    if (!entries.isEmpty()) {
        blobStore.removeBlobs(container, entries);
    }
    return entries.size();
}
 

private Set<String> getEntryNames(Predicate<? super StorageMetadata> filter) {
    return blobStore.list(container, new ListContainerOptions().withDetails())
                    .stream()
                    .filter(Objects::nonNull)
                    .filter(filter)
                    .map(StorageMetadata::getName)
                    .collect(Collectors.toSet());
}
 

private boolean filterBySpace(StorageMetadata blobMetadata, String space) {
    Map<String, String> userMetadata = blobMetadata.getUserMetadata();
    if (CollectionUtils.isEmpty(userMetadata)) {
        return false;
    }
    String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase());
    return space.equals(spaceParameter);
}
 

private boolean filterBySpaceAndNamespace(StorageMetadata blobMetadata, String space, String namespace) {
    Map<String, String> userMetadata = blobMetadata.getUserMetadata();
    if (CollectionUtils.isEmpty(userMetadata)) {
        return false;
    }
    String spaceParameter = userMetadata.get(Constants.FILE_ENTRY_SPACE.toLowerCase());
    String namespaceParameter = userMetadata.get(Constants.FILE_ENTRY_NAMESPACE.toLowerCase());
    return space.equals(spaceParameter) && namespace.equals(namespaceParameter);
}
 

@VisibleForTesting
Mono<Void> deleteAllBuckets() {
    return Flux.fromIterable(blobStore.list())
        .publishOn(Schedulers.elastic())
        .filter(storageMetadata -> storageMetadata.getType().equals(StorageType.CONTAINER))
        .map(StorageMetadata::getName)
        .doOnNext(blobStore::deleteContainer)
        .then();
}
 

@Override
public List<String> listContentsWithSubPath(final String parentSubPath) {
    checkPrepared();
    return FluentIterable.from(context.getBlobStore().list(getContainerNameFirstPart(), 
        ListContainerOptions.Builder.inDirectory(getItemInContainerSubPath(parentSubPath))))
            .transform(new Function<StorageMetadata, String>() {
                @Override
                public String apply(@javax.annotation.Nullable StorageMetadata input) {
                    String result = input.getName();
                    result = Strings.removeFromStart(result, containerSubPath);
                    result = Strings.removeFromStart(result, "/");
                    return result;
                }
            }).toList();
}
 

@Override
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
    return doOssOperation(oss -> {
        ListObjectsRequest request = toListObjectRequest(container, options);
        ObjectListing objectListing = oss.listObjects(request);
        List<StorageMetadata> storageMetadataList = toStorageMetadataList(oss, container, objectListing);
        return new PageSetImpl<>(storageMetadataList, objectListing.getNextMarker());
    });
}
 

@Override
public Map<String, App> discoverInstalledApps()
{
    Map<String, App> appMap = new HashMap<>();
    List<App> appList = new ArrayList<>();
    ObjectMapper mapper = new ObjectMapper();
    mapper.configure( DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false );

    log.info( "Starting JClouds discovery" );

    for ( StorageMetadata resource : blobStore.list( config.container, prefix( APPS_DIR + "/" ).delimiter( "/" ) ) )
    {
        log.info( "Found potential app: " + resource.getName() );

        // Found potential app
        Blob manifest = blobStore.getBlob( config.container, resource.getName() + "manifest.webapp" );

        if ( manifest == null )
        {
            log.warn( "Could not find manifest file of " + resource.getName() );
            continue;
        }

        try
        {
            InputStream inputStream = manifest.getPayload().openStream();
            App app = mapper.readValue( inputStream, App.class );
            inputStream.close();

            app.setAppStorageSource( AppStorageSource.JCLOUDS );
            app.setFolderName( resource.getName() );

            appList.add( app );
        }
        catch ( IOException ex )
        {
            log.error( "Could not read manifest file of " + resource.getName(), ex );
            log.error( DebugUtils.getStackTrace( ex ) );
        }
    }

    appList.forEach(
        app -> {
            String namespace = app.getActivities().getDhis().getNamespace();

            if ( namespace != null && !namespace.isEmpty() )
            {
                reservedNamespaces.put( namespace, app );
            }

            appMap.put( app.getUrlFriendlyName(), app );

            log.info( "Discovered app '" + app.getName() + "' from JClouds storage " );
        }
    );

    if ( appList.isEmpty() )
    {
        log.info( "No apps found during JClouds discovery." );
    }
    return appMap;
}
 

private void assertContainerFound() {
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
    BlobStoreTest.assertHasItemNamed(ps, testContainerName);
}
 

static boolean hasItemNamed(PageSet<? extends StorageMetadata> ps, String name) {
    for (StorageMetadata sm: ps)
        if (name==null || name.equals(sm.getName())) return true;
    return false;
}