org.jclouds.logging.log4j.config.Log4JLoggingModule Java Examples

/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
    String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet
        .of(new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer( bucketName );
}
 

public void deleteBucket() {

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        Blob bo = null;
        BlobStore blobStore = null;
        final Iterable<? extends Module> MODULES = ImmutableSet
                .of(new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule());

        BlobStoreContext context =
                ContextBuilder.newBuilder("s3").credentials( accessId, secretKey ).modules( MODULES )
                        .overrides( overrides ).buildView( BlobStoreContext.class );

        blobStore = context.getBlobStore();
        blobStore.deleteContainer( bucketName );
    }
 

/**
 * Delete the configured s3 bucket.
 */
public void deleteBucket() {

    logger.debug("\n\nDelete bucket\n");

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(), new NettyPayloadModule());

    BlobStoreContext context =
        ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
            .overrides(overrides ).buildView(BlobStoreContext.class);

    BlobStore blobStore = context.getBlobStore();
    blobStore.deleteContainer(bucketName);
}
 

private void copyToS3( String fileName ) {

        String bucketName = ( String ) properties.get( BUCKET_PROPNAME );
        String accessId = ( String ) properties.get( ACCESS_ID_PROPNAME );
        String secretKey = ( String ) properties.get( SECRET_KEY_PROPNAME );

        Properties overrides = new Properties();
        overrides.setProperty( "s3" + ".identity", accessId );
        overrides.setProperty( "s3" + ".credential", secretKey );

        final Iterable<? extends Module> MODULES = ImmutableSet
                .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
                        new NettyPayloadModule() );

        AWSCredentials credentials = new BasicAWSCredentials(accessId, secretKey);
        ClientConfiguration clientConfig = new ClientConfiguration();
        clientConfig.setProtocol( Protocol.HTTP);

        AmazonS3Client s3Client = new AmazonS3Client(credentials, clientConfig);

        s3Client.createBucket( bucketName );
        File uploadFile = new File( fileName );
        PutObjectResult putObjectResult = s3Client.putObject( bucketName, uploadFile.getName(), uploadFile );
        logger.info("Uploaded file etag={}", putObjectResult.getETag());
    }
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix );

        String accessId = System.getProperty(SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR);
        String secretKey = System.getProperty(SDKGlobalConfiguration.SECRET_KEY_ENV_VAR);

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for ( Object o : blobStoreList.toArray() ) {
            StorageMetadata s = (StorageMetadata)o;

            if ( s.getName().startsWith( bucketPrefix )) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch ( ContainerNotFoundException cnfe ) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe );
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

private static void deleteBucketsWithPrefix() {

        logger.debug("\n\nDelete buckets with prefix {}\n", bucketPrefix);

        String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
        String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

        Properties overrides = new Properties();
        overrides.setProperty("s3" + ".identity", accessId);
        overrides.setProperty("s3" + ".credential", secretKey);

        final Iterable<? extends Module> MODULES = ImmutableSet
            .of(new JavaUrlHttpCommandExecutorServiceModule(),
                new Log4JLoggingModule(),
                new NettyPayloadModule());

        BlobStoreContext context =
            ContextBuilder.newBuilder("s3").credentials(accessId, secretKey).modules(MODULES)
                .overrides(overrides ).buildView(BlobStoreContext.class);

        BlobStore blobStore = context.getBlobStore();
        final PageSet<? extends StorageMetadata> blobStoreList = blobStore.list();

        for (Object o : blobStoreList.toArray()) {
            StorageMetadata s = (StorageMetadata) o;

            if (s.getName().startsWith(bucketPrefix)) {
                try {
                    blobStore.deleteContainer(s.getName());
                } catch (ContainerNotFoundException cnfe) {
                    logger.warn("Attempted to delete bucket {} but it is already deleted", cnfe);
                }
                logger.debug("Deleted bucket {}", s.getName());
            }
        }
    }
 

public File copyFileFromBucket(
    String blobFileName, String bucketName, String accessId, String secretKey ) throws Exception {

    // setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // get file from configured bucket, copy it to local temp file

    Blob blob = blobStore.getBlob(bucketName, blobFileName);
    if ( blob == null) {
        throw new RuntimeException(
            "Blob file name " + blobFileName + " not found in bucket " + bucketName );
    }

    FileOutputStream fop = null;
    File tempFile;
    try {
        tempFile = File.createTempFile(bucketName, RandomStringUtils.randomAlphabetic(10));
        tempFile.deleteOnExit();
        fop = new FileOutputStream(tempFile);
        InputStream is = blob.getPayload().openStream();
        IOUtils.copyLarge(is, fop);
        return tempFile;

    } finally {
        if ( fop != null ) {
            fop.close();
        }
    }
}
 

@Override
public List<String> getBucketFileNames(
    String bucketName, String endsWith, String accessId, String secretKey ) {

    // get setup to use JCloud BlobStore interface to AWS S3

    Properties overrides = new Properties();
    overrides.setProperty("s3" + ".identity", accessId);
    overrides.setProperty("s3" + ".credential", secretKey);

    final Iterable<? extends Module> MODULES = ImmutableSet.of(
        new JavaUrlHttpCommandExecutorServiceModule(),
        new Log4JLoggingModule(),
        new NettyPayloadModule());

    BlobStoreContext context = ContextBuilder.newBuilder("s3")
        .credentials(accessId, secretKey)
        .modules(MODULES)
        .overrides(overrides)
        .buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    // gets all the files in the configured bucket recursively

    PageSet<? extends StorageMetadata> pageSets =
        blobStore.list(bucketName, new ListContainerOptions().recursive());

    if (logger.isTraceEnabled()) {
        logger.trace("   Found {} files in bucket {}", pageSets.size(), bucketName);
    }

    List<String> blobFileNames = new ArrayList<>();
    for ( Object pageSet : pageSets ) {
        String blobFileName = ((MutableBlobMetadata)pageSet).getName();
        if ( blobFileName.endsWith( endsWith )) {
            blobFileNames.add(blobFileName);
        }
    }

    return blobFileNames;
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOn() throws Exception {

    if (logger.isDebugEnabled()) {
        logger.debug("testIntegration100EntitiesOn(): starting...");
    }

    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    // create five applications each with collection of five entities

    for ( int i = 0; i < 5; i++ ) {

        ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
        EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );

        String collName = "superappCol" + i;
        appEm.createApplicationCollection(collName);

        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[5];

        for ( int index = 0; index < 5; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = appEm.create( collName, entityLevelProperties );
        }
    }

    // export the organization containing those apps and collections

    UUID exportUUID = exportService.schedule( payload );

    int maxRetries = 100;
    int retries = 0;
    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
        Thread.sleep(100);
    }

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    // test that we can find the file that were exported to S3

    BlobStore blobStore = null;
    try {

        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule());

        BlobStoreContext context = ContextBuilder.newBuilder("s3")
            .credentials(accessId, secretKey)
            .modules(MODULES)
            .overrides(overrides)
            .buildView(BlobStoreContext.class);

        String expectedFileName = ((ExportServiceImpl) exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if (!blobStore.blobExists(bucketName, expectedFileName)) {
            blobStore.deleteContainer(bucketName);
            Assert.fail("Blob does not exist: " + expectedFileName);
        }
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);

        Long numOfFiles = blobStore.countBlobs(bucketName);
        Long numWeWant = 1L;
        blobStore.deleteContainer(bucketName);
        assertEquals(numOfFiles, numWeWant);
        assertNotNull(bo);

    } finally {
        blobStore.deleteContainer(bucketName);
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesForAllApps() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 5; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
        for ( int j = 0; j < 5; j++ ) {
            appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );

            EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
            customMaker.createApplicationCollection( "superappCol" + j );
            //intialize user object to be posted
            Map<String, Object> entityLevelProperties = null;
            Entity[] entNotCopied;
            entNotCopied = new Entity[1];
            //creates entities
            for ( int index = 0; index < 1; index++ ) {
                entityLevelProperties = new LinkedHashMap<String, Object>();
                entityLevelProperties.put( "derp", "bacon" );
                entNotCopied[index] = customMaker.create( "superappCol" + j, entityLevelProperties );
            }
        }
    }

    payload.put( "organizationId", orgMade.getUuid() );

    UUID exportUUID = exportService.schedule( payload );
    assertNotNull( exportUUID );

    Thread.sleep( 3000 );

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of(
            new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(),
            new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials(accessId, secretKey )
            .modules(MODULES )
            .overrides(overrides )
            .buildView(BlobStoreContext.class );

        blobStore = context.getBlobStore();

        //Grab Number of files
        Long numOfFiles = blobStore.countBlobs( bucketName );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        //delete container containing said files
        Blob bo = blobStore.getBlob(bucketName, expectedFileName);
        Long numWeWant = 5L;
        blobStore.deleteContainer( bucketName );

        //asserts that the correct number of files was transferred over
        assertEquals( numWeWant, numOfFiles );

    }
    finally {
        blobStore.deleteContainer( bucketName );
    }
}
 

@Test
@Ignore("Pending merge of export-feature branch")
public void testIntegration100EntitiesOnOneOrg() throws Exception {

    S3Export s3Export = new S3ExportImpl();
    ExportService exportService = setup.getExportService();

    String appName = newOrgAppAdminRule.getApplicationInfo().getName();
    HashMap<String, Object> payload = payloadBuilder(appName);

    payload.put( "organizationId", organization.getUuid() );
    payload.put( "applicationId", applicationId );

    OrganizationInfo orgMade = null;
    ApplicationInfo appMade = null;
    for ( int i = 0; i < 100; i++ ) {
        orgMade = setup.getMgmtSvc().createOrganization( "largerboss" + i, adminUser, true );
        appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + i );

        EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
        customMaker.createApplicationCollection( "superappCol" + i );
        //intialize user object to be posted
        Map<String, Object> entityLevelProperties = null;
        Entity[] entNotCopied;
        entNotCopied = new Entity[20];
        //creates entities
        for ( int index = 0; index < 20; index++ ) {
            entityLevelProperties = new LinkedHashMap<String, Object>();
            entityLevelProperties.put( "username", "bobso" + index );
            entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
            entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
        }
    }

    EntityManager em = setup.getEmf().getEntityManager( applicationId );

    //intialize user object to be posted
    Map<String, Object> userProperties = null;
    Entity[] entity;
    entity = new Entity[100];

    //creates entities
    for ( int i = 0; i < 100; i++ ) {
        userProperties = new LinkedHashMap<String, Object>();
        userProperties.put( "username", "bido" + i );
        userProperties.put( "email", "bido" + i + "@anuff.com" );

        entity[i] = em.create( "user", userProperties );
    }

    UUID exportUUID = exportService.schedule( payload );

    while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}

    String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
    String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );

    Properties overrides = new Properties();
    overrides.setProperty( "s3" + ".identity", accessId );
    overrides.setProperty( "s3" + ".credential", secretKey );

    Blob bo = null;
    BlobStore blobStore = null;

    try {
        final Iterable<? extends Module> MODULES = ImmutableSet.of( new JavaUrlHttpCommandExecutorServiceModule(),
            new Log4JLoggingModule(), new NettyPayloadModule() );

        BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
            .credentials( accessId, secretKey )
            .modules( MODULES )
            .overrides( overrides )
            .buildView( BlobStoreContext.class );

        String expectedFileName = ((ExportServiceImpl)exportService)
            .prepareOutputFileName(organization.getName(), "applications");

        blobStore = context.getBlobStore();
        if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
            assert ( false );
        }
        Long numOfFiles = blobStore.countBlobs( bucketName );
        Long numWeWant = Long.valueOf( 1 );
        assertEquals( numOfFiles, numWeWant );

        bo = blobStore.getBlob( bucketName, expectedFileName );
    }
    catch ( Exception e ) {
        assert ( false );
    }

    assertNotNull( bo );
    blobStore.deleteContainer( bucketName );
}