org.jclouds.Constants Java Examples

private static BlobStoreContext blobStoreContextFromProperties(
        Properties properties) {
    String provider = properties.getProperty(Constants.PROPERTY_PROVIDER);
    String identity = properties.getProperty(Constants.PROPERTY_IDENTITY);
    String credential = properties.getProperty(
            Constants.PROPERTY_CREDENTIAL);
    String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
    if (provider == null || identity == null || credential == null) {
        System.err.println("Properties file must contain:\n" +
                Constants.PROPERTY_PROVIDER + "\n" +
                Constants.PROPERTY_IDENTITY + "\n" +
                Constants.PROPERTY_CREDENTIAL);
        System.exit(1);
    }

    ContextBuilder builder = ContextBuilder
            .newBuilder(provider)
            .credentials(identity, credential)
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .overrides(properties);
    if (endpoint != null) {
        builder = builder.endpoint(endpoint);
    }
    return builder.build(BlobStoreContext.class);
}
 

private static Pair<BlobStoreLocation, BlobStore> createBlobStore(String driver,
                                                                  String region,
                                                                  String endpoint,
                                                                  Supplier<Credentials> credentials,
                                                                  int maxBlockSize) {
    Properties overrides = new Properties();
    // This property controls the number of parts being uploaded in parallel.
    overrides.setProperty("jclouds.mpu.parallel.degree", "1");
    overrides.setProperty("jclouds.mpu.parts.size", Integer.toString(maxBlockSize));
    overrides.setProperty(Constants.PROPERTY_SO_TIMEOUT, "25000");
    overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, Integer.toString(100));

    ApiRegistry.registerApi(new S3ApiMetadata());
    ProviderRegistry.registerProvider(new AWSS3ProviderMetadata());
    ProviderRegistry.registerProvider(new GoogleCloudStorageProviderMetadata());

    ContextBuilder contextBuilder = ContextBuilder.newBuilder(driver);
    contextBuilder.credentialsSupplier(credentials);

    if (isS3Driver(driver) && !Strings.isNullOrEmpty(endpoint)) {
        contextBuilder.endpoint(endpoint);
        overrides.setProperty(S3Constants.PROPERTY_S3_VIRTUAL_HOST_BUCKETS, "false");
    }
    contextBuilder.overrides(overrides);
    BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
    BlobStore blobStore = context.getBlobStore();

    log.info("Connect to blobstore : driver: {}, region: {}, endpoint: {}",
        driver, region, endpoint);
    return Pair.of(
        BlobStoreLocation.of(region, endpoint),
        blobStore);
}
 

@Bean
@ConditionalOnProperty("openstack.endpoint")
NovaApi novaApi(@Value("${openstack.endpoint}") String endpoint,
                @Value("${openstack.tenant}") String tenant,
                @Value("${openstack.username}") String username,
                @Value("${openstack.password}") String password) {

    String identity = String.format("%s:%s", tenant, username);

    // see https://issues.apache.org/jira/browse/JCLOUDS-816
    Properties overrides = new Properties();
    overrides.put(Constants.PROPERTY_TRUST_ALL_CERTS, "true");
    overrides.put(Constants.PROPERTY_RELAX_HOSTNAME, "true");

    return ContextBuilder.newBuilder("openstack-nova")
        .endpoint(endpoint)
        .credentials(identity, password)
        .modules(Collections.singleton(new SLF4JLoggingModule()))
        .overrides(overrides)
        .buildApi(NovaApi.class);
}
 

@Override
protected Properties setupProperties() {
    TestUtils.S3ProxyLaunchInfo info;
    try {
        info = TestUtils.startS3Proxy("s3proxy.conf");
        s3Proxy = info.getS3Proxy();
        context = info.getBlobStore().getContext();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Properties props = super.setupProperties();
    props.setProperty(Constants.PROPERTY_IDENTITY, info.getS3Identity());
    props.setProperty(Constants.PROPERTY_CREDENTIAL,
            info.getS3Credential());
    props.setProperty(Constants.PROPERTY_ENDPOINT,
            info.getEndpoint().toString() + info.getServicePath());
    props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH,
            info.getServicePath());
    props.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
    endpoint = info.getEndpoint().toString();
    return props;
}
 

@Override
protected Properties setupProperties() {
    TestUtils.S3ProxyLaunchInfo info;
    try {
        info = TestUtils.startS3Proxy("s3proxy.conf");
        s3Proxy = info.getS3Proxy();
        context = info.getBlobStore().getContext();
        blobStoreType = context.unwrap().getProviderMetadata().getId();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Properties props = super.setupProperties();
    props.setProperty(Constants.PROPERTY_IDENTITY, info.getS3Identity());
    props.setProperty(Constants.PROPERTY_CREDENTIAL,
            info.getS3Credential());
    props.setProperty(Constants.PROPERTY_ENDPOINT,
            info.getEndpoint().toString() + info.getServicePath());
    props.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
    props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH,
            info.getServicePath());
    endpoint = info.getEndpoint().toString() + info.getServicePath();
    return props;
}
 

@Override
protected Properties setupProperties() {
    TestUtils.S3ProxyLaunchInfo info;
    try {
        info = TestUtils.startS3Proxy("s3proxy.conf");
        s3Proxy = info.getS3Proxy();
        context = info.getBlobStore().getContext();
        blobStoreType = context.unwrap().getProviderMetadata().getId();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Properties props = super.setupProperties();
    props.setProperty(Constants.PROPERTY_IDENTITY, info.getS3Identity());
    props.setProperty(Constants.PROPERTY_CREDENTIAL,
            info.getS3Credential());
    props.setProperty(Constants.PROPERTY_ENDPOINT,
            info.getEndpoint().toString() + info.getServicePath());
    props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH,
            info.getServicePath());
    props.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
    return props;
}
 

@Override
protected Properties setupProperties() {
    TestUtils.S3ProxyLaunchInfo info;
    try {
        info = TestUtils.startS3Proxy("s3proxy.conf");
        s3Proxy = info.getS3Proxy();
        context = info.getBlobStore().getContext();
        blobStoreType = context.unwrap().getProviderMetadata().getId();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Properties props = super.setupProperties();
    props.setProperty(Constants.PROPERTY_IDENTITY, info.getS3Identity());
    props.setProperty(Constants.PROPERTY_CREDENTIAL,
            info.getS3Credential());
    props.setProperty(Constants.PROPERTY_ENDPOINT,
            info.getEndpoint().toString() + info.getServicePath());
    props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH,
            info.getServicePath());
    props.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
    return props;
}
 

@Override
protected Properties setupProperties() {
    TestUtils.S3ProxyLaunchInfo info;
    try {
        info = TestUtils.startS3Proxy("s3proxy.conf");
        s3Proxy = info.getS3Proxy();
        context = info.getBlobStore().getContext();
        blobStoreType = context.unwrap().getProviderMetadata().getId();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    Properties props = super.setupProperties();
    props.setProperty(Constants.PROPERTY_IDENTITY, info.getS3Identity());
    props.setProperty(Constants.PROPERTY_CREDENTIAL,
            info.getS3Credential());
    props.setProperty(Constants.PROPERTY_ENDPOINT,
            info.getEndpoint().toString() + info.getServicePath());
    props.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");
    props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH,
            info.getServicePath());
    endpoint = info.getEndpoint().toString() + info.getServicePath();
    return props;
}
 

@DataProvider(name = "cloudAndImageNames")
public Object[][] cloudAndImageNames() {
    return new Object[][] {
            new Object[] { AZURE_ARM_LOCATION_SPEC, AZURE_ARM_NAME_REGEX, ImmutableMap.builder()
                    .put(Constants.PROPERTY_ENDPOINT, "https://management.azure.com/subscriptions/012e832d-7b27-4c30-9f21-22cdd9159d12")
                    .put(OAUTH_ENDPOINT.getName(), "https://login.microsoftonline.com/ba85e8cd-8c83-486e-a7e3-0d7666169d34/oauth2/token")
                    .put(JcloudsLocation.IMAGE_ID.getName(), "westeurope/MicrosoftWindowsServer/WindowsServer/2016-Datacenter")
                    .put("jclouds.azurecompute.arm.publishers", "MicrosoftWindowsServer")
                    .put("azure.arm.default.network.enabled", false)
                    .put("vmNameMaxLength", 15)
                    .put("destroyOnFailure", false)
                    .put("useJcloudsSshInit", false)
                    .build()
            },
            new Object[] { AWS_EC2_LOCATION_SPEC, AWS_EC2_IMAGE_NAME_REGEX, ImmutableMap.of() },
            new Object[] { GCE_LOCATION_SPEC, GCE_IMAGE_NAME_REGEX, ImmutableMap.of(JcloudsLocation.LOGIN_USER.getName(), "myname") },
        };
}
 

private void resetBlobstore(String username, String password, String provider, String region) {
   Properties overrides = new Properties();
   // Retry after 25 seconds of no response
   overrides.setProperty(Constants.PROPERTY_SO_TIMEOUT, "25000");
   // Keep retrying indefinitely
   overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, String.valueOf(Integer.MAX_VALUE));
   // Do not wait between retries
   overrides.setProperty(Constants.PROPERTY_RETRY_DELAY_START , "0");

   ContextBuilder builder = ContextBuilder.newBuilder(provider)
         .overrides(overrides)
         .credentials(username, password);
   blobStore.set(builder.buildView(RegionScopedBlobStoreContext.class).getBlobStore(region));
}
 

@Inject
protected AWSEC2ListNodesStrategy(AWSEC2Api client, @Region Supplier<Set<String>> regions,
         Function<RunningInstance, NodeMetadata> runningInstanceToNodeMetadata,
         @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor,
         SpotInstanceRequestToAWSRunningInstance spotConverter) {
   super(client, regions, runningInstanceToNodeMetadata, userExecutor);
   this.client = checkNotNull(client, "client");
   this.spotConverter = checkNotNull(spotConverter, "spotConverter");
}
 

@Inject
public AWSEC2SecurityGroupExtension(AWSEC2Api client,
                                    @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor,
                                    @Region Supplier<Set<String>> regions,
                                    Function<org.jclouds.ec2.domain.SecurityGroup, SecurityGroup> groupConverter,
                                    @Memoized Supplier<Set<? extends Location>> locations,
                                    @Named("SECURITY") LoadingCache<RegionAndName, String> groupCreator,
                                    GroupNamingConvention.Factory namingConvention,
                                    @Named("SECGROUP_NAME_TO_ID") Function<String, String> groupNameToId) {
   super(client, userExecutor, regions, groupConverter, locations, groupCreator, namingConvention);
   this.client = checkNotNull(client, "client");
   this.groupNameToId = checkNotNull(groupNameToId, "groupNameToId");
}
 

@Inject
protected AWSEC2ComputeService(ComputeServiceContext context, Map<String, Credentials> credentialStore,
      @Memoized Supplier<Set<? extends Image>> images, @Memoized Supplier<Set<? extends Hardware>> sizes,
      @Memoized Supplier<Set<? extends Location>> locations, ListNodesStrategy listNodesStrategy,
      GetImageStrategy getImageStrategy, GetNodeMetadataStrategy getNodeMetadataStrategy,
      CreateNodesInGroupThenAddToSet runNodesAndAddToSetStrategy, RebootNodeStrategy rebootNodeStrategy,
      DestroyNodeStrategy destroyNodeStrategy, ResumeNodeStrategy startNodeStrategy,
      SuspendNodeStrategy stopNodeStrategy, Provider<TemplateBuilder> templateBuilderProvider,
      @Named("DEFAULT") Provider<TemplateOptions> templateOptionsProvider,
      @Named(TIMEOUT_NODE_RUNNING) Predicate<AtomicReference<NodeMetadata>> nodeRunning,
      @Named(TIMEOUT_NODE_TERMINATED) Predicate<AtomicReference<NodeMetadata>> nodeTerminated,
      @Named(TIMEOUT_NODE_SUSPENDED) Predicate<AtomicReference<NodeMetadata>> nodeSuspended,
      InitializeRunScriptOnNodeOrPlaceInBadMap.Factory initScriptRunnerFactory,
      RunScriptOnNode.Factory runScriptOnNodeFactory, InitAdminAccess initAdminAccess,
      PersistNodeCredentials persistNodeCredentials, Timeouts timeouts,
      @Named(Constants.PROPERTY_USER_THREADS) ListeningExecutorService userExecutor, AWSEC2Api client,
      ConcurrentMap<RegionAndName, KeyPair> credentialsMap,
      @Named("SECURITY") LoadingCache<RegionAndName, String> securityGroupMap,
      @Named("PLACEMENT") LoadingCache<RegionAndName, String> placementGroupMap,
      @Named("DELETED") Predicate<PlacementGroup> placementGroupDeleted, Optional<ImageExtension> imageExtension,
      GroupNamingConvention.Factory namingConvention,
      @Named(PROPERTY_EC2_GENERATE_INSTANCE_NAMES) boolean generateInstanceNames,
      Optional<SecurityGroupExtension> securityGroupExtension) {
   super(context, credentialStore, images, sizes, locations, listNodesStrategy, getImageStrategy,
         getNodeMetadataStrategy, runNodesAndAddToSetStrategy, rebootNodeStrategy, destroyNodeStrategy,
         startNodeStrategy, stopNodeStrategy, templateBuilderProvider, templateOptionsProvider, nodeRunning,
         nodeTerminated, nodeSuspended, initScriptRunnerFactory, runScriptOnNodeFactory, initAdminAccess,
         persistNodeCredentials, timeouts, userExecutor, client, credentialsMap, securityGroupMap, imageExtension,
         namingConvention, generateInstanceNames, securityGroupExtension);
   this.client = client;
   this.placementGroupMap = placementGroupMap;
   this.placementGroupDeleted = placementGroupDeleted;
}
 

protected ContextBuilder builder(Properties overrides) {
   MockWebServer defaultServer = regionToServers.get(DEFAULT_REGION);
   overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, "1");
   overrides.setProperty(ComputeServiceProperties.TIMEOUT_CLEANUP_INCIDENTAL_RESOURCES, "0");
   return ContextBuilder.newBuilder(new AWSEC2ProviderMetadata())
         .credentials(ACCESS_KEY, SECRET_KEY)
         .endpoint(defaultServer.getUrl("").toString())
         .overrides(overrides)
         .modules(modules);
}
 

public static Map<Integer, Integer> dockerPortMappingsFor(JcloudsLocation docker, String containerId) {
    ComputeServiceContext context = null;
    try {
        Properties properties = new Properties();
        properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
        properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
        context = ContextBuilder.newBuilder("docker")
                .endpoint(docker.getEndpoint())
                .credentials(docker.getIdentity(), docker.getCredential())
                .overrides(properties)
                .modules(ImmutableSet.<Module>of(new SLF4JLoggingModule(), new SshjSshClientModule()))
                .build(ComputeServiceContext.class);
        DockerApi api = context.unwrapApi(DockerApi.class);
        Container container = api.getContainerApi().inspectContainer(containerId);
        Map<Integer, Integer> portMappings = Maps.newLinkedHashMap();
        Map<String, List<Map<String, String>>> ports = container.networkSettings().ports();
        if (ports == null) ports = ImmutableMap.<String, List<Map<String,String>>>of();

        LOG.debug("Docker will forward these ports {}", ports);
        for (Map.Entry<String, List<Map<String, String>>> entrySet : ports.entrySet()) {
            String containerPort = Iterables.get(Splitter.on("/").split(entrySet.getKey()), 0);
            String hostPort = Iterables.getOnlyElement(Iterables.transform(entrySet.getValue(),
                    new Function<Map<String, String>, String>() {
                        @Override
                        public String apply(Map<String, String> hostIpAndPort) {
                            return hostIpAndPort.get("HostPort");
                        }
                    }));
            portMappings.put(Integer.parseInt(containerPort), Integer.parseInt(hostPort));
        }
        return portMappings;
    } finally {
        if (context != null) {
            context.close();
        }
    }
}
 

private Map<?, ?> makeCacheKey(ConfigBag conf, Properties properties) {
    String provider = getProviderFromConfig(conf);
    String identity = checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
    String credential = checkNotNull(conf.get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");
    String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
    return MutableMap.builder()
            .putAll(properties)
            .put("provider", provider)
            .put("identity", identity)
            .put("credential", credential)
            .putIfNotNull("endpoint", endpoint)
            .build()
            .asUnmodifiable();
}
 

@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
    JCloudsPropertiesBuilder propertiesBuilder = new JCloudsPropertiesBuilder(conf)
            .setCommonJcloudsProperties();

    // Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise customize for specific owners; or all as a last resort
    // See https://issues.apache.org/jira/browse/WHIRR-416
    String provider = getProviderFromConfig(conf);
    if ("aws-ec2".equals(provider)) {
        propertiesBuilder.setAWSEC2Properties();
    } else if ("azurecompute-arm".equals(provider)) {
        propertiesBuilder.setAzureComputeArmProperties();
    }

    Properties properties = propertiesBuilder
            .setCustomJcloudsProperties()
            .setEndpointProperty()
            .build();

    ImmutableSet.Builder<? extends Module> modulesBuilder = ImmutableSet.<Module>builder()
            .addAll(commonModules())
            .addAll(userDefinedModules(conf))
            .addAll(linkingContext(conf));

    Iterable<? extends Module> modules = modulesBuilder.build();

    //Check for missing endpoint when provider is azure
    if (properties.getProperty(Constants.PROPERTY_ENDPOINT) == null && ("azurecompute-arm".equals(provider))) {
        throw new IllegalArgumentException("Endpoint property cannot be null when provider is azure-arm");
    }

    Supplier<ComputeService> computeServiceSupplier = new ComputeServiceSupplier(conf, modules, properties);
    if (allowReuse) {
        return cachedComputeServices.computeIfAbsent(makeCacheKey(conf, properties), key -> computeServiceSupplier.get());
    }
    return computeServiceSupplier.get();
}
 

@Override
public BlobStoreContext newBlobStoreContext(Location location) {
    String rawProvider = checkNotNull(location.getConfig(LocationConfigKeys.CLOUD_PROVIDER), "provider must not be null");
    String provider = DeserializingJcloudsRenamesProvider.INSTANCE.applyJcloudsRenames(rawProvider);
    String identity = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_IDENTITY), "identity must not be null");
    String credential = checkNotNull(location.getConfig(LocationConfigKeys.ACCESS_CREDENTIAL), "credential must not be null");
    String endpoint = location.getConfig(CloudLocationConfig.CLOUD_ENDPOINT);

    Properties overrides = new Properties();
    // * Java 7,8 bug workaround - sockets closed by GC break the internal bookkeeping
    //   of HttpUrlConnection, leading to invalid handling of the "HTTP/1.1 100 Continue"
    //   response. Coupled with a bug when using SSL sockets reads will block
    //   indefinitely even though a read timeout is explicitly set.
    // * Java 6 ignores the header anyways as it is included in its restricted headers black list.
    // * Also there's a bug in SL object store which still expects Content-Length bytes
    //   even when it responds with a 408 timeout response, leading to incorrectly
    //   interpreting the next request (triggered by above problem).
    overrides.setProperty(Constants.PROPERTY_STRIP_EXPECT_HEADER, "true");

    // Add extra jclouds-specific configuration
    Map<String, Object> extra = Maps.filterKeys(((LocationInternal)location).config().getBag().getAllConfig(), Predicates.containsPattern("^jclouds\\."));
    if (extra.size() > 0) {
        LOG.debug("Configuring custom jclouds property overrides for {}: {}", provider, Sanitizer.sanitize(extra));
    }
    overrides.putAll(Maps.filterValues(extra, Predicates.notNull()));

    ContextBuilder contextBuilder = ContextBuilder.newBuilder(provider).credentials(identity, credential);
    contextBuilder.modules(MutableList.copyOf(getCommonModules()));
    if (!org.apache.brooklyn.util.text.Strings.isBlank(endpoint)) {
        contextBuilder.endpoint(endpoint);
    }
    contextBuilder.overrides(overrides);
    BlobStoreContext context = contextBuilder.buildView(BlobStoreContext.class);
    return context;
}
 

public Ec2Context(Ec2Credentials credentials) {
  this.credentials = credentials;
  Properties properties = new Properties();
  long scriptTimeout = TimeUnit.MILLISECONDS.convert(50, TimeUnit.MINUTES);
  properties.setProperty(TIMEOUT_SCRIPT_COMPLETE, scriptTimeout + "");
  properties.setProperty(TIMEOUT_PORT_OPEN, scriptTimeout + "");
  properties.setProperty(PROPERTY_CONNECTION_TIMEOUT, scriptTimeout + "");
  properties.setProperty(PROPERTY_EC2_AMI_QUERY, "owner-id=137112412989;state=available;image-type=machine");
  properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
  properties.setProperty(Constants.PROPERTY_MAX_RETRIES, Settings.JCLOUDS_PROPERTY_MAX_RETRIES + "");
  properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, Settings.JCLOUDS_PROPERTY_RETRY_DELAY_START + "");

  Iterable<Module> modules = ImmutableSet.<Module>of(
      new SshjSshClientModule(),
      new SLF4JLoggingModule(),
      new EnterpriseConfigurationModule());

  ContextBuilder build = ContextBuilder.newBuilder("aws-ec2")
      .credentials(credentials.getAccessKey(), credentials.getSecretKey())
      .modules(modules)
      .overrides(properties);
  ComputeServiceContext context = build.buildView(ComputeServiceContext.class);
  this.computeService = (AWSEC2ComputeService) context.getComputeService();
  this.ec2api = computeService.getContext().unwrapApi(EC2Api.class);
  this.securityGroupApi = ec2api.getSecurityGroupApi().get();
  this.keypairApi = (AWSKeyPairApi) ec2api.getKeyPairApi().get();

  vmBatchSize = Settings.AWS_VM_BATCH_SIZE();
}
 

@Override
protected Properties setupProperties() {
   Properties overrides = super.setupProperties();
   overrides.put(Constants.PROPERTY_API_VERSION, "2014-05-01");
   return overrides;
}
 

@Override
protected Properties setupProperties() {
   Properties overrides = super.setupProperties();
   overrides.put(Constants.PROPERTY_API_VERSION, "2014-05-01");
   return overrides;
}
 

public static void main(String[] args) throws Exception {
    Options options = new Options();
    CmdLineParser parser = new CmdLineParser(options);
    try {
        parser.parseArgument(args);
    } catch (CmdLineException cle) {
        PrintStream err = System.err;
        err.println("are-we-consistent-yet version " +
                AreWeConsistentYet.class.getPackage()
                        .getImplementationVersion());
        err.println("Usage: are-we-consistent-yet" +
                " --container-name NAME --properties FILE [options...]");
        parser.printUsage(err);
        System.exit(1);
    }

    Properties properties = new Properties();
    try (InputStream is = new FileInputStream(options.propertiesFile)) {
        properties.load(is);
    }
    Properties propertiesRead = (Properties) properties.clone();
    if (options.readerEndpoint != null) {
        propertiesRead.setProperty(Constants.PROPERTY_ENDPOINT,
                options.readerEndpoint);
    }

    try (BlobStoreContext context = blobStoreContextFromProperties(
                 properties);
         BlobStoreContext contextRead = blobStoreContextFromProperties(
                 propertiesRead)) {
        BlobStore blobStore = context.getBlobStore();
        BlobStore blobStoreRead = contextRead.getBlobStore();

        Location location = null;
        if (options.location != null) {
            for (Location loc : blobStore.listAssignableLocations()) {
                if (loc.getId().equalsIgnoreCase(options.location)) {
                    location = loc;
                    break;
                }
            }
            if (location == null) {
                throw new Exception("Could not find location: " +
                        options.location);
            }
        }
        blobStore.createContainerInLocation(location,
                options.containerName);
        AreWeConsistentYet test = new AreWeConsistentYet(
                blobStore, blobStoreRead, options.containerName,
                options.iterations, options.objectSize);
        PrintStream out = System.out;
        out.println("eventual consistency count with " +
                options.iterations + " iterations: ");
        out.println("read after create: " + test.readAfterCreate());
        out.println("read after delete: " + test.readAfterDelete());
        out.println("read after overwrite: " + test.readAfterOverwrite());
        out.println("list after create: " + test.listAfterCreate());
        out.println("list after delete: " + test.listAfterDelete());
        blobStore.deleteContainer(options.containerName);
    }
}
 

static S3ProxyLaunchInfo startS3Proxy(String configFile) throws Exception {
    S3ProxyLaunchInfo info = new S3ProxyLaunchInfo();

    try (InputStream is = Resources.asByteSource(Resources.getResource(
            configFile)).openStream()) {
        info.getProperties().load(is);
    }

    String provider = info.getProperties().getProperty(
            Constants.PROPERTY_PROVIDER);
    String identity = info.getProperties().getProperty(
            Constants.PROPERTY_IDENTITY);
    String credential = info.getProperties().getProperty(
            Constants.PROPERTY_CREDENTIAL);
    if (provider.equals("google-cloud-storage")) {
        File credentialFile = new File(credential);
        if (credentialFile.exists()) {
            credential = Files.asCharSource(credentialFile,
                    StandardCharsets.UTF_8).read();
        }
        info.getProperties().remove(Constants.PROPERTY_CREDENTIAL);
    }
    String endpoint = info.getProperties().getProperty(
            Constants.PROPERTY_ENDPOINT);

    ContextBuilder builder = ContextBuilder
            .newBuilder(provider)
            .credentials(identity, credential)
            .modules(ImmutableList.<Module>of(new SLF4JLoggingModule()))
            .overrides(info.getProperties());
    if (!Strings.isNullOrEmpty(endpoint)) {
        builder.endpoint(endpoint);
    }
    BlobStoreContext context = builder.build(BlobStoreContext.class);
    info.blobStore = context.getBlobStore();

    S3Proxy.Builder s3ProxyBuilder = S3Proxy.Builder.fromProperties(
            info.getProperties());
    s3ProxyBuilder.blobStore(info.blobStore);
    info.endpoint = s3ProxyBuilder.getEndpoint();
    info.secureEndpoint = s3ProxyBuilder.getSecureEndpoint();
    info.s3Identity = s3ProxyBuilder.getIdentity();
    info.s3Credential = s3ProxyBuilder.getCredential();
    info.servicePath = s3ProxyBuilder.getServicePath();
    info.getProperties().setProperty(Constants.PROPERTY_USER_AGENT,
            String.format("s3proxy/%s jclouds/%s java/%s",
                    TestUtils.class.getPackage().getImplementationVersion(),
                    JcloudsVersion.get(),
                    System.getProperty("java.version")));

    // resolve relative path for tests
    String keyStorePath = info.getProperties().getProperty(
            S3ProxyConstants.PROPERTY_KEYSTORE_PATH);
    String keyStorePassword = info.getProperties().getProperty(
            S3ProxyConstants.PROPERTY_KEYSTORE_PASSWORD);
    if (keyStorePath != null || keyStorePassword != null) {
        s3ProxyBuilder.keyStore(
                Resources.getResource(keyStorePath).toString(),
                keyStorePassword);
    }
    info.s3Proxy = s3ProxyBuilder.build();
    info.s3Proxy.start();
    while (!info.s3Proxy.getState().equals(AbstractLifeCycle.STARTED)) {
        Thread.sleep(1);
    }

    // reset endpoint to handle zero port
    info.endpoint = new URI(info.endpoint.getScheme(),
            info.endpoint.getUserInfo(), info.endpoint.getHost(),
            info.s3Proxy.getPort(), info.endpoint.getPath(),
            info.endpoint.getQuery(), info.endpoint.getFragment());
    if (info.secureEndpoint != null) {
        info.secureEndpoint = new URI(info.secureEndpoint.getScheme(),
                info.secureEndpoint.getUserInfo(),
                info.secureEndpoint.getHost(),
                info.s3Proxy.getSecurePort(),
                info.secureEndpoint.getPath(),
                info.secureEndpoint.getQuery(),
                info.secureEndpoint.getFragment());
    }

    return info;
}
 

@Override
protected Properties setupProperties() {
    Properties overrides = super.setupProperties();
    overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, "0");
    return overrides;
}
 

private static BlobStore createBlobStore(Properties properties,
        ExecutorService executorService) throws IOException {
    String provider = properties.getProperty(Constants.PROPERTY_PROVIDER);
    String identity = properties.getProperty(Constants.PROPERTY_IDENTITY);
    String credential = properties.getProperty(
            Constants.PROPERTY_CREDENTIAL);
    String endpoint = properties.getProperty(Constants.PROPERTY_ENDPOINT);
    properties.remove(Constants.PROPERTY_ENDPOINT);
    String region = properties.getProperty(
            LocationConstants.PROPERTY_REGION);

    if (provider == null) {
        System.err.println(
                "Properties file must contain: " +
                Constants.PROPERTY_PROVIDER);
        System.exit(1);
    }

    if (provider.equals("filesystem") || provider.equals("transient")) {
        // local blobstores do not require credentials
        identity = Strings.nullToEmpty(identity);
        credential = Strings.nullToEmpty(credential);
    } else if (provider.equals("google-cloud-storage")) {
        File credentialFile = new File(credential);
        if (credentialFile.exists()) {
            credential = Files.asCharSource(credentialFile,
                    StandardCharsets.UTF_8).read();
        }
        properties.remove(Constants.PROPERTY_CREDENTIAL);
    }

    if (identity == null || credential == null) {
        System.err.println(
                "Properties file must contain: " +
                Constants.PROPERTY_IDENTITY + " and " +
                Constants.PROPERTY_CREDENTIAL);
        System.exit(1);
    }

    properties.setProperty(Constants.PROPERTY_USER_AGENT,
            String.format("s3proxy/%s jclouds/%s java/%s",
                    Main.class.getPackage().getImplementationVersion(),
                    JcloudsVersion.get(),
                    System.getProperty("java.version")));

    ContextBuilder builder = ContextBuilder
            .newBuilder(provider)
            .credentials(identity, credential)
            .modules(ImmutableList.<Module>of(
                    new SLF4JLoggingModule(),
                    new ExecutorServiceModule(executorService)))
            .overrides(properties);
    if (!Strings.isNullOrEmpty(endpoint)) {
        builder = builder.endpoint(endpoint);
    }

    BlobStoreContext context = builder.build(BlobStoreContext.class);
    BlobStore blobStore;
    if (context instanceof RegionScopedBlobStoreContext &&
            region != null) {
        blobStore = ((RegionScopedBlobStoreContext) context)
                .getBlobStore(region);
    } else {
        blobStore = context.getBlobStore();
    }
    return blobStore;
}
 

public JCloudsPropertiesBuilder setEndpointProperty() {
    String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
    if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
    if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
    return this;
}
 

public JCloudsPropertiesBuilder setCommonJcloudsProperties() {
    properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
    properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
    properties.setProperty("jclouds.ssh.max-retries", conf.getStringKey("jclouds.ssh.max-retries") != null ?
            conf.getStringKey("jclouds.ssh.max-retries").toString() : "50");

    if (conf.get(OAUTH_ENDPOINT) != null)
        properties.setProperty(OAUTH_ENDPOINT.getName(), conf.get(OAUTH_ENDPOINT));

    /*
     * See https://issues.apache.org/jira/browse/BROOKLYN-394
     * For retries, the backoff times are:
     *   Math.min(2^failureCount * retryDelayStart, retryDelayStart * 10) + random(10%)
     * The overridden Brooklyn defaults give backoffs of:
     *   2s, 4s, 8s, 16s, 20s, 20s, 20s, 20s, 20s, 20s
     * (i.e. we'll keep trying for approx 2mins 30secs).
     * 
     * The jclouds defaults (if not overridden here) are 50ms and 5 retires. This gives backoff
     * times of:
     *   50ms, 100ms, 200ms, 400ms, 500ms
     * That is only a total of 1.25s, which is not long when you're being rate-limited 
     * and there are multiple thread all retrying their API calls.
     * 
     * Further improvements/considerations include:
     *  - Users with huge deployments may want to backoff for even longer, e.g. setting:
     *      jclouds.retries-delay-start: 3s
     *      jclouds.max-retries: 20
     *  - The jitter could be improved to include more randomness, but that is not yet 
     *    configurable in jclouds. See:
     *     - https://aws.amazon.com/blogs/architecture/exponential-backoff-and-jitter/
     *     - https://docs.aws.amazon.com/general/latest/gr/api-retries.html
     *     - https://github.com/jclouds/jclouds/blob/rel/jclouds-2.0.2/core/src/main/java/org/jclouds/http/handlers/BackoffLimitedRetryHandler.java#L135-L136
     */
    properties.setProperty(Constants.PROPERTY_RETRY_DELAY_START, "2000");
    properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "10");
    
    /*
     * Controls polling for the VM status as 'running', when provisioning a new VM.
     * The backoff times are calculated as:
     *   Math.min(1.5^(attempt-1) * initialPeriod, maxPeriod)
     * See:
     *   https://github.com/jclouds/jclouds/blob/rel/jclouds-2.0.2/core/src/main/java/org/jclouds/util/Predicates2.java#L157-L172
     * The overridden Brooklyn defaults give backoffs of:
     *   1s, 1.5s, 2s, 2s, ...
     *  
     * Defaults in jclouds are initial 50ms, backing off to max 1s. That would give backoffs of:
     *   50ms, 75ms, 112ms, 169ms, 253ms, 380ms, 570ms, 854ms, 1s, ...
     * i.e. approx 5 calls in the first second, and another 2 calls in the next second, etc.
     * For detecting completion of VM provisioning (which even in a fast cloud currently 
     * takes > 10s), that is far more load than is sensible with default settings.
     */
    properties.setProperty(ComputeServiceProperties.POLL_INITIAL_PERIOD, "1000");
    properties.setProperty(ComputeServiceProperties.POLL_MAX_PERIOD, "2000");

    return this;
}
 

@Override
protected Properties setupProperties() {
    final Properties overrides = super.setupProperties();
    overrides.setProperty(Constants.PROPERTY_MAX_RETRIES, "5");
    return overrides;
}
 

protected Properties setupProperties() {
    final Properties properties = new Properties();
    properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "0");
    return properties;
}
 

protected Properties setupProperties() {
    final Properties properties = new Properties();
    properties.setProperty(Constants.PROPERTY_MAX_RETRIES, "0");
    return properties;
}