Java Code Examples for org.jclouds.blobstore.BlobStore#getBlob()

public static ReadHandle open(ScheduledExecutorService executor,
                              BlobStore blobStore, String bucket, String key, String indexKey,
                              VersionCheck versionCheck,
                              long ledgerId, int readBufferSize)
        throws IOException {
    Blob blob = blobStore.getBlob(bucket, indexKey);
    versionCheck.check(indexKey, blob);
    OffloadIndexBlockBuilder indexBuilder = OffloadIndexBlockBuilder.create();
    OffloadIndexBlock index = indexBuilder.fromStream(blob.getPayload().openStream());

    BackedInputStream inputStream = new BlobStoreBackedInputStreamImpl(blobStore, bucket, key,
        versionCheck,
        index.getDataObjectLength(),
        readBufferSize);
    return new BlobStoreBackedReadHandleImpl(ledgerId, index, inputStream, executor);
}
 

private Blob get(String name) {
    try {
        BlobStore blobStore = context.getBlobStore();
        return blobStore.getBlob(containerName, name);
    } catch (Exception e) {
        LOG.error("GET " + name + " failed", e);
        return null;
    }
}
 

@Override
protected void doCall(BlobStore blobStore, String indent) throws Exception {
    Blob content = blobStore.getBlob(container, blob);
    stdout.println(indent+"Blob "+container+" : " +blob +" {");
    stdout.println(indent+"\tHeaders {");
    for (Map.Entry<String, String> entry : content.getAllHeaders().entries()) {
        stdout.println(indent+"\t\t"+entry.getKey() + " = " + entry.getValue());
    }
    stdout.println(indent+"\t}");
    stdout.println(indent+"\tmetadata : "+content.getMetadata());
    stdout.println(indent+"\tpayload : "+ Streams.readFullyStringAndClose(content.getPayload().openStream()));
    stdout.println(indent+"}");
}
 

private static void putAndRetrieveBlobExample(BlobStore blobstore) throws IOException {
   // Create a container
   String containerName = "jclouds_putAndRetrieveBlobExample_" + UUID.randomUUID().toString();
   blobstore.createContainerInLocation(null, containerName); // Create a vault

   // Create a blob
   ByteSource payload = ByteSource.wrap("data".getBytes(Charsets.UTF_8));
   Blob blob = blobstore.blobBuilder("ignored") // The blob name is ignored in Glacier
         .payload(payload)
         .contentLength(payload.size())
         .build();

   // Put the blob in the container
   String blobId = blobstore.putBlob(containerName, blob);

   // Retrieve the blob
   Blob result = blobstore.getBlob(containerName, blobId);

   // Print the result
   InputStream is = result.getPayload().openStream();
   try {
      String data = CharStreams.toString(new InputStreamReader(is, Charsets.UTF_8));
      System.out.println("The retrieved payload is: " + data);
   } finally {
      is.close();
   }
}
 

private void handleGetBlob(HttpServletRequest request,
        HttpServletResponse response, BlobStore blobStore,
        String containerName, String blobName)
        throws IOException, S3Exception {
    int status = HttpServletResponse.SC_OK;
    GetOptions options = new GetOptions();

    String ifMatch = request.getHeader(HttpHeaders.IF_MATCH);
    if (ifMatch != null) {
        options.ifETagMatches(ifMatch);
    }

    String ifNoneMatch = request.getHeader(HttpHeaders.IF_NONE_MATCH);
    if (ifNoneMatch != null) {
        options.ifETagDoesntMatch(ifNoneMatch);
    }

    long ifModifiedSince = request.getDateHeader(
            HttpHeaders.IF_MODIFIED_SINCE);
    if (ifModifiedSince != -1) {
        options.ifModifiedSince(new Date(ifModifiedSince));
    }

    long ifUnmodifiedSince = request.getDateHeader(
            HttpHeaders.IF_UNMODIFIED_SINCE);
    if (ifUnmodifiedSince != -1) {
        options.ifUnmodifiedSince(new Date(ifUnmodifiedSince));
    }

    String range = request.getHeader(HttpHeaders.RANGE);
    if (range != null && range.startsWith("bytes=") &&
            // ignore multiple ranges
            range.indexOf(',') == -1) {
        range = range.substring("bytes=".length());
        String[] ranges = range.split("-", 2);
        if (ranges[0].isEmpty()) {
            options.tail(Long.parseLong(ranges[1]));
        } else if (ranges[1].isEmpty()) {
            options.startAt(Long.parseLong(ranges[0]));
        } else {
            options.range(Long.parseLong(ranges[0]),
                    Long.parseLong(ranges[1]));
        }
        status = HttpServletResponse.SC_PARTIAL_CONTENT;
    }

    Blob blob = blobStore.getBlob(containerName, blobName, options);
    if (blob == null) {
        throw new S3Exception(S3ErrorCode.NO_SUCH_KEY);
    }

    response.setStatus(status);

    String corsOrigin = request.getHeader(HttpHeaders.ORIGIN);
    if (!Strings.isNullOrEmpty(corsOrigin) &&
            corsRules.isOriginAllowed(corsOrigin)) {
        response.addHeader(HttpHeaders.ACCESS_CONTROL_ALLOW_ORIGIN,
                corsOrigin);
        response.addHeader(HttpHeaders.ACCESS_CONTROL_ALLOW_METHODS, "GET");
    }

    addMetadataToResponse(request, response, blob.getMetadata());
    // TODO: handles only a single range due to jclouds limitations
    Collection<String> contentRanges =
            blob.getAllHeaders().get(HttpHeaders.CONTENT_RANGE);
    if (!contentRanges.isEmpty()) {
        response.addHeader(HttpHeaders.CONTENT_RANGE,
                contentRanges.iterator().next());
        response.addHeader(HttpHeaders.ACCEPT_RANGES,
                "bytes");
    }

    try (InputStream is = blob.getPayload().openStream();
         OutputStream os = response.getOutputStream()) {
        ByteStreams.copy(is, os);
        os.flush();
    }
}
 

public File copyFileFromBucket(
    String blobFileName, String bucketName, String accessId, String secretKey ) throws Exception {

    // setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // get file from configured bucket, copy it to local temp file

    Blob blob = blobStore.getBlob(bucketName, blobFileName);
    if ( blob == null) {
        throw new RuntimeException(
            "Blob file name " + blobFileName + " not found in bucket " + bucketName );
    }

    FileOutputStream fop = null;
    File tempFile;
    try {
        tempFile = File.createTempFile(bucketName, RandomStringUtils.randomAlphabetic(10));
        tempFile.deleteOnExit();
        fop = new FileOutputStream(tempFile);
        InputStream is = blob.getPayload().openStream();
        IOUtils.copyLarge(is, fop);
        return tempFile;

    } finally {
        if ( fop != null ) {
            fop.close();
        }
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOn() throws Exception {

    if (logger.isDebugEnabled()) {
        logger.debug("testIntegration100EntitiesOn(): starting...");
    }

    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    // create five applications each with collection of five entities

    for ( int i = 0; i < 5; i++ ) {

        ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
        EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );

        String collName = "superappCol" + i;
        appEm.createApplicationCollection(collName);

        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[5];

        for ( int index = 0; index < 5; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = appEm.create( collName, entityLevelProperties );
        }
    }

    // export the organization containing those apps and collections

    UUID exportUUID = exportService.schedule( payload );

    int maxRetries = 100;
    int retries = 0;
    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
        Thread.sleep(100);
    }

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    // test that we can find the file that were exported to S3

    BlobStore blobStore = null;
    try {

        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

        BlobStoreContext context = ContextBuilder.newBuilder("s3")
            .credentials(accessId, secretKey)
            .modules(MODULES)
            .overrides(overrides)
            .buildView(BlobStoreContext.class);

        String expectedFileName = ((ExportServiceImpl) exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if (!blobStore.blobExists(bucketName, expectedFileName)) {
            blobStore.deleteContainer(bucketName);
            Assert.fail("Blob does not exist: " + expectedFileName);
        }
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);

        Long numOfFiles = blobStore.countBlobs(bucketName);
        Long numWeWant = 1L;
        blobStore.deleteContainer(bucketName);
        assertEquals(numOfFiles, numWeWant);
        assertNotNull(bo);

    } finally {
        blobStore.deleteContainer(bucketName);
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesForAllApps() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 5; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
        for ( int j = 0; j < 5; j++ ) {
            appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );

            EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
            customMaker.createApplicationCollection( "superappCol" + j );
            //intialize user object to be posted
            Map<String, Object> entityLevelProperties = null;
            Entity[] entNotCopied;
            entNotCopied = new Entity[1];
            //creates entities
            for ( int index = 0; index < 1; index++ ) {
                entityLevelProperties = new LinkedHashMap<String, Object>();
                entityLevelProperties.put( "derp", "bacon" );
                entNotCopied[index] = customMaker.create( "superappCol" + j, entityLevelProperties );
            }
        }
    }

    payload.put( "organizationId", orgMade.getUuid() );

    UUID exportUUID = exportService.schedule( payload );
    assertNotNull( exportUUID );

    Thread.sleep( 3000 );

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials(accessId, secretKey )
            .modules(MODULES )
            .overrides(overrides )
            .buildView(BlobStoreContext.class );

        blobStore = context.getBlobStore();

        //Grab Number of files
        Long numOfFiles = blobStore.countBlobs( bucketName );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        //delete container containing said files
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);
        Long numWeWant = 5L;
        blobStore.deleteContainer( bucketName );

        //asserts that the correct number of files was transferred over
        assertEquals( numWeWant, numOfFiles );

    }
    finally {
        blobStore.deleteContainer( bucketName );
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOnOneOrg() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 100; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "largerboss" + i, adminUser, true );
        appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + i );

        EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
        customMaker.createApplicationCollection( "superappCol" + i );
        //intialize user object to be posted
        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[20];
        //creates entities
        for ( int index = 0; index < 20; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
        }
    }

    EntityManager em = setup.getEmf().getEntityManager( applicationId );

    //intialize user object to be posted
    Map<String, Object> userProperties = null;
    Entity[] entity;
    entity = new Entity[100];

    //creates entities
    for ( int i = 0; i < 100; i++ ) {
        userProperties = new LinkedHashMap<String, Object>();
        userProperties.put( "username", "bido" + i );
        userProperties.put( "email", "bido" + i + "@anuff.com" );

        entity[i] = em.create( "user", userProperties );
    }

    UUID exportUUID = exportService.schedule( payload );

    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    Blob bo = null;
    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(), new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials( accessId, secretKey )
            .modules( MODULES )
            .overrides( overrides )
            .buildView( BlobStoreContext.class );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
            assert ( false );
        }
        Long numOfFiles = blobStore.countBlobs( bucketName );
        Long numWeWant = Long.valueOf( 1 );
        assertEquals( numOfFiles, numWeWant );

        bo = blobStore.getBlob( bucketName, expectedFileName );
    }
    catch ( Exception e ) {
        assert ( false );
    }

    assertNotNull( bo );
    blobStore.deleteContainer( bucketName );
}