org.apache.hadoop.fs.s3a.Constants Java Examples

The following examples show how to use org.apache.hadoop.fs.s3a.Constants. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@Override
protected void setup(Configuration conf) throws IOException, RuntimeException {
  try {
    s3 = clientCache.get(S3ClientKey.create(conf));
    useWhitelistedBuckets = !conf.get(S3StoragePlugin.WHITELISTED_BUCKETS,"").isEmpty();
    if (!NONE_PROVIDER.equals(conf.get(Constants.AWS_CREDENTIALS_PROVIDER))
      && !conf.getBoolean(COMPATIBILITY_MODE, false)) {
      verifyCredentials(conf);
    }
  } catch (ExecutionException e) {
    Throwable cause = e.getCause();
    if (cause == null) {
      throw new RuntimeException(e);
    }

    Throwables.throwIfInstanceOf(cause, IOException.class);
    Throwables.throwIfUnchecked(cause);

    throw new RuntimeException(cause);
  }
}
 
Example #2
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
protected AwsCredentialsProvider getAsync2Provider(Configuration config) {
  switch(config.get(Constants.AWS_CREDENTIALS_PROVIDER)) {
    case ACCESS_KEY_PROVIDER:
      return StaticCredentialsProvider.create(AwsBasicCredentials.create(
        config.get(Constants.ACCESS_KEY), config.get(Constants.SECRET_KEY)));
    case EC2_METADATA_PROVIDER:
      return InstanceProfileCredentialsProvider.create();
    case NONE_PROVIDER:
      return AnonymousCredentialsProvider.create();
    case ASSUME_ROLE_PROVIDER:
      return new STSCredentialProviderV2(config);
    default:
      throw new IllegalStateException(config.get(Constants.AWS_CREDENTIALS_PROVIDER));
  }
}
 
Example #3
Source File: STSCredentialProviderV1.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
public STSCredentialProviderV1(URI uri, Configuration conf) throws IOException {

    AWSCredentialsProvider awsCredentialsProvider = null;

    //TODO: Leverage S3AUtils createAwsCredentialProvider

    if (S3StoragePlugin.ACCESS_KEY_PROVIDER.equals(conf.get(Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER))) {
      awsCredentialsProvider = new SimpleAWSCredentialsProvider(uri, conf);
    } else if (S3StoragePlugin.EC2_METADATA_PROVIDER.equals(conf.get(Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER))) {
      awsCredentialsProvider = InstanceProfileCredentialsProvider.getInstance();
    }

    final String region = S3FileSystem.getAWSRegionFromConfigurationOrDefault(conf).toString();
    final AWSSecurityTokenServiceClientBuilder builder = AWSSecurityTokenServiceClientBuilder.standard()
      .withCredentials(awsCredentialsProvider)
      .withClientConfiguration(S3AUtils.createAwsConf(conf, ""))
      .withRegion(region);
    S3FileSystem.getStsEndpoint(conf).ifPresent(e -> {
      builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(e, region));
    });

    this.stsAssumeRoleSessionCredentialsProvider = new STSAssumeRoleSessionCredentialsProvider.Builder(
      conf.get(Constants.ASSUMED_ROLE_ARN), UUID.randomUUID().toString())
      .withStsClient(builder.build())
      .build();
  }
 
Example #4
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
@Override
protected Stream<ContainerCreator> getContainerCreators() throws IOException {
  Stream<String> buckets = getBucketNamesFromConfigurationProperty(S3StoragePlugin.EXTERNAL_BUCKETS);
  if (!NONE_PROVIDER.equals(getConf().get(Constants.AWS_CREDENTIALS_PROVIDER))) {
    if (!useWhitelistedBuckets) {
      // if we have authentication to access S3, add in owner buckets.
      buckets = Stream.concat(buckets, s3.listBuckets().stream().map(Bucket::getName));
    } else {
      // Only add the buckets provided in the configuration.
      buckets = Stream.concat(buckets, getBucketNamesFromConfigurationProperty(S3StoragePlugin.WHITELISTED_BUCKETS));
    }
  }
  return buckets.distinct() // Remove duplicate bucket names.
      .map(input -> new BucketCreator(getConf(), input));
}
 
Example #5
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
static software.amazon.awssdk.regions.Region getAWSRegionFromConfigurationOrDefault(Configuration conf) {
  final String regionOverride = conf.getTrimmed(REGION_OVERRIDE);
  if (!Strings.isNullOrEmpty(regionOverride)) {
    // set the region to what the user provided unless they provided an empty string.
    return software.amazon.awssdk.regions.Region.of(regionOverride);
  }

  return getAwsRegionFromEndpoint(conf.get(Constants.ENDPOINT));
}
 
Example #6
Source File: STSCredentialProviderV2.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public STSCredentialProviderV2(Configuration conf) {
  AwsCredentialsProvider awsCredentialsProvider = null;

  if (S3StoragePlugin.ACCESS_KEY_PROVIDER.equals(conf.get(Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER))) {
    awsCredentialsProvider = StaticCredentialsProvider.create(AwsBasicCredentials.create(
      conf.get(Constants.ACCESS_KEY), conf.get(Constants.SECRET_KEY)));
  } else if (S3StoragePlugin.EC2_METADATA_PROVIDER.equals(conf.get(Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER))) {
    awsCredentialsProvider = InstanceProfileCredentialsProvider.create();
  }

  final StsClientBuilder builder = StsClient.builder()
    .credentialsProvider(awsCredentialsProvider)
    .region(S3FileSystem.getAWSRegionFromConfigurationOrDefault(conf))
    .httpClientBuilder(initConnectionSettings(conf));
  S3FileSystem.getStsEndpoint(conf).ifPresent(e -> {
    try {
      builder.endpointOverride(new URI(e));
    } catch (URISyntaxException use) {
      throw UserException.sourceInBadState(use).buildSilently();
    }
  });

  initUserAgent(builder, conf);

  final AssumeRoleRequest assumeRoleRequest = AssumeRoleRequest.builder()
    .roleArn(conf.get(Constants.ASSUMED_ROLE_ARN))
    .roleSessionName(UUID.randomUUID().toString())
    .build();

  this.stsAssumeRoleCredentialsProvider = StsAssumeRoleCredentialsProvider.builder()
    .refreshRequest(assumeRoleRequest)
    .stsClient(builder.build())
    .build();
}
 
Example #7
Source File: STSCredentialProviderV2.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public static SdkHttpClient.Builder initConnectionSettings(Configuration conf) {
  final ApacheHttpClient.Builder httpBuilder = ApacheHttpClient.builder();
  httpBuilder.maxConnections(intOption(conf, Constants.MAXIMUM_CONNECTIONS, Constants.DEFAULT_MAXIMUM_CONNECTIONS, 1));
  httpBuilder.connectionTimeout(
    Duration.ofSeconds(intOption(conf, Constants.ESTABLISH_TIMEOUT, Constants.DEFAULT_ESTABLISH_TIMEOUT, 0)));
  httpBuilder.socketTimeout(
    Duration.ofSeconds(intOption(conf, Constants.SOCKET_TIMEOUT, Constants.DEFAULT_SOCKET_TIMEOUT, 0)));
  httpBuilder.proxyConfiguration(initProxySupport(conf));

  return httpBuilder;
}
 
Example #8
Source File: STSCredentialProviderV2.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public static ProxyConfiguration initProxySupport(Configuration conf) throws IllegalArgumentException {
  final ProxyConfiguration.Builder builder = ProxyConfiguration.builder();

  final String proxyHost = conf.getTrimmed(Constants.PROXY_HOST, "");
  int proxyPort = conf.getInt(Constants.PROXY_PORT, -1);
  if (!proxyHost.isEmpty()) {
    if (proxyPort < 0) {
      if (conf.getBoolean(Constants.SECURE_CONNECTIONS, Constants.DEFAULT_SECURE_CONNECTIONS)) {
        proxyPort = 443;
      } else {
        proxyPort = 80;
      }
    }

    builder.endpoint(URI.create(proxyHost + ":" + proxyPort));

    try {
      final String proxyUsername = lookupPassword(conf, Constants.PROXY_USERNAME);
      final String proxyPassword = lookupPassword(conf, Constants.PROXY_PASSWORD);
      if ((proxyUsername == null) != (proxyPassword == null)) {
        throw new IllegalArgumentException(String.format("Proxy error: %s or %s set without the other.",
          Constants.PROXY_USERNAME, Constants.PROXY_PASSWORD));
      }

      builder.username(proxyUsername);
      builder.password(proxyPassword);
      builder.ntlmDomain(conf.getTrimmed(Constants.PROXY_DOMAIN));
      builder.ntlmWorkstation(conf.getTrimmed(Constants.PROXY_WORKSTATION));
    } catch (IOException e) {
      throw UserException.sourceInBadState(e).buildSilently();
    }
  } else if (proxyPort >= 0) {
    throw new IllegalArgumentException(String.format("Proxy error: %s set without %s",
      Constants.PROXY_HOST, Constants.PROXY_PORT));
  }

  return builder.build();
}
 
Example #9
Source File: STSCredentialProviderV2.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
private static void initUserAgent(StsClientBuilder builder, Configuration conf) {
  String userAgent = "Hadoop " + VersionInfo.getVersion();
  final String userAgentPrefix = conf.getTrimmed(Constants.USER_AGENT_PREFIX, "");
  if (!userAgentPrefix.isEmpty()) {
    userAgent = userAgentPrefix + ", " + userAgent;
  }

  builder.overrideConfiguration(ClientOverrideConfiguration.builder()
    .putAdvancedOption(SdkAdvancedClientOption.USER_AGENT_PREFIX, userAgent)
    .build());
}
 
Example #10
Source File: FileUtil.java    From secor with Apache License 2.0 5 votes vote down vote up
public static void configure(SecorConfig config) {
    if (config != null) {
        if (config.getCloudService().equals("Swift")) {
            mConf.set("fs.swift.impl", "org.apache.hadoop.fs.swift.snative.SwiftNativeFileSystem");
            mConf.set("fs.swift.service.GENERICPROJECT.auth.url", config.getSwiftAuthUrl());
            mConf.set("fs.swift.service.GENERICPROJECT.username", config.getSwiftUsername());
            mConf.set("fs.swift.service.GENERICPROJECT.tenant", config.getSwiftTenant());
            mConf.set("fs.swift.service.GENERICPROJECT.region", config.getSwiftRegion());
            mConf.set("fs.swift.service.GENERICPROJECT.http.port", config.getSwiftPort());
            mConf.set("fs.swift.service.GENERICPROJECT.use.get.auth", config.getSwiftGetAuth());
            mConf.set("fs.swift.service.GENERICPROJECT.public", config.getSwiftPublic());
            if (config.getSwiftGetAuth().equals("true")) {
                mConf.set("fs.swift.service.GENERICPROJECT.apikey", config.getSwiftApiKey());
            } else {
                mConf.set("fs.swift.service.GENERICPROJECT.password", config.getSwiftPassword());
            }
        } else if (config.getCloudService().equals("S3")) {
            if (config.getAwsAccessKey().isEmpty() != config.getAwsSecretKey().isEmpty()) {
                throw new IllegalArgumentException(
                    "Must specify both aws.access.key and aws.secret.key or neither.");
            }
            if (!config.getAwsAccessKey().isEmpty()) {
                mConf.set(Constants.ACCESS_KEY, config.getAwsAccessKey());
                mConf.set(Constants.SECRET_KEY, config.getAwsSecretKey());
                mConf.set("fs.s3n.awsAccessKeyId", config.getAwsAccessKey());
                mConf.set("fs.s3n.awsSecretAccessKey", config.getAwsSecretKey());
            }
        }
    }
}
 
Example #11
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public String toString() {
  return "[ Access Key=" + s3Config.get(Constants.ACCESS_KEY) + ", Secret Key =*****, isSecure=" +
      s3Config.get(SECURE_CONNECTIONS) + " ]";
}
 
Example #12
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
static Optional<String> getEndpoint(Configuration conf) {
  return Optional.ofNullable(conf.getTrimmed(Constants.ENDPOINT))
    .map(s -> getHttpScheme(conf) + s);
}
 
Example #13
Source File: S3FileSystem.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
static Optional<String> getStsEndpoint(Configuration conf) {
  return Optional.ofNullable(conf.getTrimmed(Constants.ASSUMED_ROLE_STS_ENDPOINT))
    .map(s -> "https://" + s);
}
 
Example #14
Source File: S3StoragePlugin.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
protected List<Property> getProperties() {
  final S3PluginConfig config = getConfig();
  final List<Property> finalProperties = new ArrayList<>();
  finalProperties.add(new Property(org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY, "dremioS3:///"));
  finalProperties.add(new Property("fs.dremioS3.impl", S3FileSystem.class.getName()));
  finalProperties.add(new Property(MAXIMUM_CONNECTIONS, String.valueOf(DEFAULT_MAX_CONNECTIONS)));
  finalProperties.add(new Property(FAST_UPLOAD, "true"));
  finalProperties.add(new Property(Constants.FAST_UPLOAD_BUFFER, "disk"));
  finalProperties.add(new Property(Constants.FAST_UPLOAD_ACTIVE_BLOCKS, "4")); // 256mb (so a single parquet file should be able to flush at once).
  finalProperties.add(new Property(MAX_THREADS, "24"));
  finalProperties.add(new Property(MULTIPART_SIZE, "67108864")); // 64mb
  finalProperties.add(new Property(MAX_TOTAL_TASKS, "30"));

  if(config.compatibilityMode) {
    finalProperties.add(new Property(S3FileSystem.COMPATIBILITY_MODE, "true"));
  }

  String mainAWSCredProvider;

  switch (config.credentialType) {
    case ACCESS_KEY:
      if (("".equals(config.accessKey)) || ("".equals(config.accessSecret))) {
        throw UserException.validationError()
          .message("Failure creating S3 connection. You must provide AWS Access Key and AWS Access Secret.")
          .build(logger);
      }
      mainAWSCredProvider = ACCESS_KEY_PROVIDER;
      finalProperties.add(new Property(ACCESS_KEY, config.accessKey));
      finalProperties.add(new Property(SECRET_KEY, config.accessSecret));
      break;
    case EC2_METADATA:
      mainAWSCredProvider = EC2_METADATA_PROVIDER;
      break;
    case NONE:
      mainAWSCredProvider = NONE_PROVIDER;
      break;
    default:
      throw new RuntimeException("Failure creating S3 connection. Invalid credentials type.");
  }

  if (!Strings.isNullOrEmpty(config.assumedRoleARN) && !NONE_PROVIDER.equals(mainAWSCredProvider)) {
    finalProperties.add(new Property(Constants.ASSUMED_ROLE_ARN, config.assumedRoleARN));
    finalProperties.add(new Property(Constants.ASSUMED_ROLE_CREDENTIALS_PROVIDER, mainAWSCredProvider));
    mainAWSCredProvider = ASSUME_ROLE_PROVIDER;
  }

  finalProperties.add(new Property(Constants.AWS_CREDENTIALS_PROVIDER, mainAWSCredProvider));

  final List<Property> propertyList = super.getProperties();
  if (propertyList != null && !propertyList.isEmpty()) {
    finalProperties.addAll(propertyList);
  }

  finalProperties.add(new Property(SECURE_CONNECTIONS, String.valueOf(config.secure)));
  if (config.externalBucketList != null && !config.externalBucketList.isEmpty()) {
    finalProperties.add(new Property(EXTERNAL_BUCKETS, Joiner.on(",").join(config.externalBucketList)));
  } else {
    if (config.credentialType == AWSAuthenticationType.NONE) {
      throw UserException.validationError()
        .message("Failure creating S3 connection. You must provide one or more external buckets when you choose no authentication.")
        .build(logger);
    }
  }

  if (config.whitelistedBuckets != null && !config.whitelistedBuckets.isEmpty()) {
    finalProperties.add(new Property(WHITELISTED_BUCKETS, Joiner.on(",").join(config.whitelistedBuckets)));
  }

  if (!Strings.isNullOrEmpty(config.kmsKeyARN)) {
    finalProperties.add(new Property(SERVER_SIDE_ENCRYPTION_KEY, config.kmsKeyARN));
    finalProperties.add(new Property(SERVER_SIDE_ENCRYPTION_ALGORITHM, S3AEncryptionMethods.SSE_KMS.getMethod()));
    finalProperties.add(new Property(SECURE_CONNECTIONS, Boolean.TRUE.toString()));
  }

  finalProperties.add(new Property(ALLOW_REQUESTER_PAYS, Boolean.toString(config.requesterPays)));

  return finalProperties;
}