org.jclouds.blobstore.domain.PageSet Java Examples

private NavigableMap<Long,String> getBlobsWithinRange(PageSet<? extends StorageMetadata> pages) {
    // TreeMap used because of sorted property
    TreeMap<Long, String> tsToBlobName = new TreeMap<Long, String>();
    for (StorageMetadata blobMeta : pages) {
        String fileName = blobMeta.getName(); // 20140226_1393442533000.json.gz
        String dateAndTs = fileName.split("\\.", 2)[0].trim(); // 20140226_1393442533000
        String tsCreated = dateAndTs.split("_")[1].trim(); // 1393442533000
        long ts = Long.parseLong(tsCreated);
        tsToBlobName.put(ts, fileName);
    }
    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = tsToBlobName.subMap(START_TIME - 60000*15, true, STOP_TIME + 60000*30, true);
    if(mapWithinRange.isEmpty()) {
        lastMarker = tsToBlobName.lastEntry().getValue().trim();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(tsToBlobName.lastEntry().getValue().trim());
        }
    }
    return mapWithinRange;
}
 

public void testCreateListDestroySimpleDirInContainer() throws IOException {
    context.getBlobStore().createContainerInLocation(null, testContainerName);
    context.getBlobStore().createDirectory(testContainerName, "my-dir-1");
    
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list(testContainerName);
    assertHasItemNamed(ps, "my-dir-1");
    
    Blob b = context.getBlobStore().blobBuilder("my-blob-1").payload(Streams.newInputStreamWithContents("hello world")).build();
    context.getBlobStore().putBlob(testContainerName+"/"+"my-dir-1", b);
    
    ps = context.getBlobStore().list(testContainerName, ListContainerOptions.Builder.inDirectory("my-dir-1"));
    assertHasItemNamed(ps, "my-dir-1/my-blob-1");

    // both these syntaxes work:
    Blob b2 = context.getBlobStore().getBlob(testContainerName+"/"+"my-dir-1", "my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b2.getPayload().openStream()), "hello world");

    Blob b3 = context.getBlobStore().getBlob(testContainerName, "my-dir-1"+"/"+"my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b3.getPayload().openStream()), "hello world");

    context.getBlobStore().deleteContainer(testContainerName);
}
 

public synchronized boolean hasNewFiles() {
    // see if there are any files since lastMarker.
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);
    
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);
    
    log.debug("Saw {} new files since {}", pages.size() == batchSize ? "many" : Integer.toString(pages.size()), lastMarker);
    boolean emptiness = getBlobsWithinRange(pages).isEmpty();

    if(emptiness) {
        log.warn("No file found within range {}", new Range(START_TIME, STOP_TIME));
    } else {
        log.debug("New files found within range {}", new Range(START_TIME, STOP_TIME));
    }

    return !emptiness;
}
 

private Set<String> listAllBlobs() {
    Set<String> blobNames = new HashSet<String>();
    ListContainerOptions options = new ListContainerOptions();
    while (true) {
        PageSet<? extends StorageMetadata> set = blobStoreRead.list(
                containerName, options);
        for (StorageMetadata sm : set) {
            blobNames.add(sm.getName());
        }
        String marker = set.getNextMarker();
        if (marker == null) {
            break;
        }
        options = options.afterMarker(marker);
    }
    return blobNames;
}
 

@Test
public void testCreateBlobGetBlob() throws Exception {
    String blobName = createRandomBlobName();
    Blob blob = makeBlob(nullBlobStore, blobName);
    nullBlobStore.putBlob(containerName, blob);

    blob = nullBlobStore.getBlob(containerName, blobName);
    validateBlobMetadata(blob.getMetadata());

    // content differs, only compare length
    try (InputStream actual = blob.getPayload().openStream();
            InputStream expected = BYTE_SOURCE.openStream()) {
        long actualLength = ByteStreams.copy(actual,
                ByteStreams.nullOutputStream());
        long expectedLength = ByteStreams.copy(expected,
                ByteStreams.nullOutputStream());
        assertThat(actualLength).isEqualTo(expectedLength);
    }

    PageSet<? extends StorageMetadata> pageSet = nullBlobStore.list(
            containerName);
    assertThat(pageSet).hasSize(1);
    StorageMetadata sm = pageSet.iterator().next();
    assertThat(sm.getName()).isEqualTo(blobName);
    assertThat(sm.getSize()).isEqualTo(0);
}
 

@Override
public PageSet<? extends StorageMetadata> list(String container) {
    ImmutableSet.Builder<StorageMetadata> builder = ImmutableSet.builder();
    PageSet<? extends StorageMetadata> pageSet = super.list(container);
    for (StorageMetadata sm : pageSet) {
        MutableStorageMetadata msm = new MutableStorageMetadataImpl(sm);
        msm.setSize(0L);
        builder.add(msm);
    }
    return new PageSetImpl<>(builder.build(), pageSet.getNextMarker());
}
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

public synchronized void downloadNewFiles(File downloadDir) {
    log.info("Downloading new files since {}", lastMarker);
    
    BlobStoreContext ctx = ContextBuilder.newBuilder(provider)
            .credentials(user, key)
            .overrides(new Properties() {{
                setProperty(LocationConstants.PROPERTY_ZONE, zone);
            }})
            .buildView(BlobStoreContext.class);

    // threadsafe according to https://jclouds.apache.org/documentation/userguide/blobstore-guide/
    BlobStore store = ctx.getBlobStore();
    ListContainerOptions options = new ListContainerOptions().maxResults(batchSize).afterMarker(lastMarker);
    PageSet<? extends StorageMetadata> pages = store.list(container, options);

    //Gets key within the time range specified
    NavigableMap<Long, String> mapWithinRange = getBlobsWithinRange(pages);

    //Download only for keys within that range
    for(Map.Entry<Long, String> blobMeta : mapWithinRange.entrySet()) {
        log.info("Downloading file: " + blobMeta.getValue());
        downloadWorkers.submit(new BlobDownload(downloadDir, store, container, blobMeta.getValue()));
        lastMarker = blobMeta.getValue();
        synchronized (CloudFilesManager.this) {
            // this is where we resume from.
            MarkerUtils.writeLastMarker(blobMeta.getValue());
        }
    }
    log.info("Updated the last marker value as " + lastMarker);
}
 

@Override
public PageSet<? extends StorageMetadata> list(String container, ListContainerOptions options) {
    return doOssOperation(oss -> {
        ListObjectsRequest request = toListObjectRequest(container, options);
        ObjectListing objectListing = oss.listObjects(request);
        List<StorageMetadata> storageMetadataList = toStorageMetadataList(oss, container, objectListing);
        return new PageSetImpl<>(storageMetadataList, objectListing.getNextMarker());
    });
}
 

public void testCreateListDestroyContainer() throws IOException {
    context.getBlobStore().createContainerInLocation(null, testContainerName);
    context.getBlobStore().list(testContainerName);
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
    assertHasItemNamed(ps, testContainerName);
    
    Blob b = context.getBlobStore().blobBuilder("my-blob-1").payload(Streams.newInputStreamWithContents("hello world")).build();
    context.getBlobStore().putBlob(testContainerName, b);
    
    Blob b2 = context.getBlobStore().getBlob(testContainerName, "my-blob-1");
    Assert.assertEquals(Streams.readFullyStringAndClose(b2.getPayload().openStream()), "hello world");
    
    context.getBlobStore().deleteContainer(testContainerName);
}
 

public static void main(String[] args) {
        LocalManagementContextForTests mgmt = new LocalManagementContextForTests(BrooklynProperties.Factory.newDefault()); 
        JcloudsLocation location = (JcloudsLocation) mgmt.getLocationRegistry().getLocationManaged(locationSpec);
            
        String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null");
        String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null");
        String provider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
        String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);

        BlobStoreContext context = ContextBuilder.newBuilder(provider)
                .credentials(identity, credential)
                .endpoint(endpoint)
                .buildView(BlobStoreContext.class);
        
        PageSet<? extends StorageMetadata> containers = context.getBlobStore().list();
        for (StorageMetadata container: containers) {
            if (container.getName().matches("brooklyn.*-test.*")
                // to kill all containers here
//                || container.getName().matches(".*")
                ) {
                log.info("killing - "+container.getName());
                context.getBlobStore().deleteContainer(container.getName());
            } else {
                log.info("KEEPING - "+container.getName());
            }
        }
        context.close();
    }
 

static boolean hasItemNamed(PageSet<? extends StorageMetadata> ps, String name) {
    for (StorageMetadata sm: ps)
        if (name==null || name.equals(sm.getName())) return true;
    return false;
}
 

static void assertHasItemNamed(PageSet<? extends StorageMetadata> ps, String name) {
    if (!hasItemNamed(ps, name))
        Assert.fail("No item named '"+name+"' in "+ps);
}
 

@Override
public List<String> getBucketFileNames(
    String bucketName, String endsWith, String accessId, String secretKey ) {

    // get setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // gets all the files in the configured bucket recursively

    PageSet<? extends StorageMetadata> pageSets =
        blobStore.list(bucketName, new ListContainerOptions().recursive());

    if (logger.isTraceEnabled()) {
        logger.trace("   Found {} files in bucket {}", pageSets.size(), bucketName);
    }

    List<String> blobFileNames = new ArrayList<>();
    for ( Object pageSet : pageSets ) {
        String blobFileName = ((MutableBlobMetadata)pageSet).getName();
        if ( blobFileName.endsWith( endsWith )) {
            blobFileNames.add(blobFileName);
        }
    }

    return blobFileNames;
}
 

private void assertContainerFound() {
    PageSet<? extends StorageMetadata> ps = context.getBlobStore().list();
    BlobStoreTest.assertHasItemNamed(ps, testContainerName);
}
 

@Override
public PageSet<? extends StorageMetadata> list() {
    throw new UnsupportedOperationException();
}