com.amazonaws.services.kinesis.AmazonKinesisClientBuilder Java Examples

The following examples show how to use com.amazonaws.services.kinesis.AmazonKinesisClientBuilder. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AWSUtil.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 
Example #2
Source File: AWSUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// Set signingRegion as null, to facilitate mocking Kinesis for local tests
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
												configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
												null));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 
Example #3
Source File: AWSUtil.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an Amazon Kinesis Client.
 * @param configProps configuration properties containing the access key, secret key, and region
 * @param awsClientConfig preconfigured AWS SDK client configuration
 * @return a new Amazon Kinesis Client
 */
public static AmazonKinesis createKinesisClient(Properties configProps, ClientConfiguration awsClientConfig) {
	// set a Flink-specific user agent
	awsClientConfig.setUserAgentPrefix(String.format(USER_AGENT_FORMAT,
			EnvironmentInformation.getVersion(),
			EnvironmentInformation.getRevisionInformation().commitId));

	// utilize automatic refreshment of credentials by directly passing the AWSCredentialsProvider
	AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder.standard()
			.withCredentials(AWSUtil.getCredentialsProvider(configProps))
			.withClientConfiguration(awsClientConfig);

	if (configProps.containsKey(AWSConfigConstants.AWS_ENDPOINT)) {
		// If an endpoint is specified, we give preference to using an endpoint and use the region property to
		// sign the request.
		builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(
			configProps.getProperty(AWSConfigConstants.AWS_ENDPOINT),
			configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	} else {
		builder.withRegion(Regions.fromName(configProps.getProperty(AWSConfigConstants.AWS_REGION)));
	}
	return builder.build();
}
 
Example #4
Source File: AWSKinesisLocalContainerService.java    From camel-kafka-connector with Apache License 2.0 5 votes vote down vote up
@Override
public AmazonKinesis getClient() {
    ClientConfiguration clientConfiguration = new ClientConfiguration();
    clientConfiguration.setProtocol(Protocol.HTTP);

    return AmazonKinesisClientBuilder
            .standard()
            .withEndpointConfiguration(getContainer().getEndpointConfiguration(LocalStackContainer.Service.KINESIS))
            .withCredentials(getContainer().getDefaultCredentialsProvider())
            .withClientConfiguration(clientConfiguration)
            .build();
}
 
Example #5
Source File: AWSClientUtils.java    From camel-kafka-connector with Apache License 2.0 5 votes vote down vote up
public static AmazonKinesis newKinesisClient() {
    LOG.debug("Creating a new AWS Kinesis client");
    AmazonKinesisClientBuilder clientBuilder = AmazonKinesisClientBuilder.standard();

    String awsInstanceType = System.getProperty("aws-service.kinesis.instance.type");
    String region = getRegion();

    if (awsInstanceType == null || awsInstanceType.equals("local-aws-container")) {
        String amazonHost = System.getProperty(AWSConfigs.AMAZON_AWS_HOST);

        LOG.debug("Creating a new AWS Kinesis client to access {}", amazonHost);

        ClientConfiguration clientConfiguration = new ClientConfiguration();
        clientConfiguration.setProtocol(Protocol.HTTP);

        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(amazonHost, region))
                .withClientConfiguration(clientConfiguration)
                .withCredentials(new TestAWSCredentialsProvider("accesskey", "secretkey"));
    } else {
        clientBuilder
            .withRegion(region)
            .withCredentials(new TestAWSCredentialsProvider());
    }

    return clientBuilder.build();
}
 
Example #6
Source File: AmazonDockerClientsHolder.java    From spring-localstack with Apache License 2.0 5 votes vote down vote up
@Override
public AmazonKinesis amazonKinesis() {
    return decorateWithConfigsAndBuild(
        AmazonKinesisClientBuilder.standard(),
        LocalstackDocker::getEndpointKinesis
    );
}
 
Example #7
Source File: HaystackKinesisForwarderTest.java    From pitchfork with Apache License 2.0 5 votes vote down vote up
private static AmazonKinesis setupKinesisClient() {
    var endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(KINESIS_SERVICE_ENDPOINT, "us-west-1");

    return AmazonKinesisClientBuilder.standard()
            .withCredentials(kinesisContainer.getDefaultCredentialsProvider())
            .withEndpointConfiguration(endpointConfiguration)
            .build();
}
 
Example #8
Source File: BasicKinesisProvider.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
public AmazonKinesis getKinesisClient() {
  AmazonKinesisClientBuilder clientBuilder =
      AmazonKinesisClientBuilder.standard().withCredentials(getCredentialsProvider());
  if (serviceEndpoint == null) {
    clientBuilder.withRegion(region);
  } else {
    clientBuilder.withEndpointConfiguration(
        new AwsClientBuilder.EndpointConfiguration(serviceEndpoint, region.getName()));
  }
  return clientBuilder.build();
}
 
Example #9
Source File: KinesisUtil.java    From datacollector with Apache License 2.0 5 votes vote down vote up
private static AmazonKinesis getKinesisClient(ClientConfiguration awsClientConfig, KinesisConfigBean conf) throws StageException {

    AmazonKinesisClientBuilder builder = AmazonKinesisClientBuilder
        .standard()
        .withClientConfiguration(checkNotNull(awsClientConfig))
        .withCredentials(AWSUtil.getCredentialsProvider(conf.awsConfig));

    if (AwsRegion.OTHER == conf.region) {
      builder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(conf.endpoint, null));
    } else {
      builder.withRegion(conf.region.getId());
    }

    return builder.build();
  }
 
Example #10
Source File: TalendKinesisProvider.java    From components with Apache License 2.0 5 votes vote down vote up
@Override
public AmazonKinesis getKinesisClient() {
    AmazonKinesisClientBuilder clientBuilder =
            AmazonKinesisClientBuilder.standard().withCredentials(getCredentialsProvier());
    if (specifyEndpoint) {
        clientBuilder
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, region.getName()));
    } else {
        clientBuilder.setRegion(region.getName());
    }
    return clientBuilder.build();
}
 
Example #11
Source File: KinesisPubsubClient.java    From flink with Apache License 2.0 5 votes vote down vote up
private static AmazonKinesis createClientWithCredentials(Properties props) throws AmazonClientException {
	AWSCredentialsProvider credentialsProvider = new EnvironmentVariableCredentialsProvider();
	return AmazonKinesisClientBuilder.standard()
		.withCredentials(credentialsProvider)
		.withEndpointConfiguration(
			new AwsClientBuilder.EndpointConfiguration(
				props.getProperty(ConsumerConfigConstants.AWS_ENDPOINT), "us-east-1"))
		.build();
}
 
Example #12
Source File: KinesisUtils.java    From wildfly-camel with Apache License 2.0 5 votes vote down vote up
public static AmazonKinesisClient createKinesisClient() {
    BasicCredentialsProvider credentials = BasicCredentialsProvider.standard();
    AmazonKinesisClient client = !credentials.isValid() ? null : (AmazonKinesisClient)
            AmazonKinesisClientBuilder.standard()
            .withCredentials(credentials)
            .withRegion("eu-west-1").build();
    return client;
}
 
Example #13
Source File: KinesisConfig.java    From bidder with Apache License 2.0 4 votes vote down vote up
public KinesisConfig(String address) throws Exception {

    	boolean create = true;
        // Chop off kineses://
        address = address.substring(10);

        // Substitute from environment variales;
        address = Configuration.substitute(address);

        String parts[] = address.split("&");
        for (int i = 0; i < parts.length; i++) {
            String t[] = parts[i].trim().split("=");
            if (t.length != 2) {
                throw new Exception("Not a proper parameter (a=b)");
            }
            t[0] = t[0].trim();
            t[1] = t[1].trim();
            switch (t[0]) {
                case "aws_access_key":
                    aws_access_key = t[1].trim();
                    break;
                case "aws_secret_key":
                    aws_secret_key = t[1].trim();
                    break;
                case "aws_region":
                    aws_region = t[1].trim();
                    break;
                case "topic":
                case "stream":
                    stream = t[1].trim();
                    break;
                case "partition":
                    partition = t[1].trim();
                    break;
                case "iterator_type":
                    iterator_type = t[1].trim();
                    break;
                case "record_limit":
                    record_limit = Integer.parseInt(t[1].trim());
                    break;
                case "sleep_time":
                    sleep_time = Integer.parseInt(t[1].trim());
                    break;
                case "create":
                    create = Boolean.parseBoolean(t[1].trim());
                    break;
            }
        }

        if (aws_access_key == null)
            throw new Exception("aws_access_key is not defined");

        if (aws_secret_key == null)
            throw new Exception("aws_secret_key is not defined");

        if (stream == null)
            throw new Exception("No stream defined");

        awsCreds = new BasicAWSCredentials(aws_access_key, aws_secret_key);

        amazonKinesis = AmazonKinesisClientBuilder
                .standard().withRegion(Regions.fromName(aws_region))
                .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
                .build();
        
        if (create) {
            createTopicIfNotActive(stream);
        }
    }
 
Example #14
Source File: LambdaKinesisSdkWriteHandler.java    From Serverless-Programming-Cookbook with MIT License 4 votes vote down vote up
public LambdaKinesisSdkWriteHandler() {
    this.kinesisClient = AmazonKinesisClientBuilder.standard()
            .withRegion(System.getenv("AWS_REGION"))
            .build();
}
 
Example #15
Source File: KinesisInventoryUtil.java    From pacbot with Apache License 2.0 4 votes vote down vote up
public static Map<String,List<DataStreamVH>> fetchDataStreamInfo(BasicSessionCredentials temporaryCredentials, String skipRegions,String accountId,String accountName) {
    
    Map<String,List<DataStreamVH>> dataStream = new LinkedHashMap<>();
    AmazonKinesis amazonKinesis;
    String expPrefix = InventoryConstants.ERROR_PREFIX_CODE+accountId + "\",\"Message\": \"Exception in fetching info for resource\" ,\"type\": \"datastream\"" ;
    for(Region region : RegionUtils.getRegions()) { 
        try{
            if(!skipRegions.contains(region.getName())){
                amazonKinesis = AmazonKinesisClientBuilder.standard().withCredentials(new AWSStaticCredentialsProvider(temporaryCredentials)).withRegion(region.getName()).build();
                ListStreamsResult listStreamsResult = amazonKinesis.listStreams();
                List<String> streamNamesTemp = listStreamsResult.getStreamNames();
                List<String> streamNames = new ArrayList<>(streamNamesTemp);
                while (listStreamsResult.isHasMoreStreams() && !streamNamesTemp.isEmpty()) {
                    listStreamsResult = amazonKinesis.listStreams(streamNamesTemp.get(streamNamesTemp.size() - 1));
                    streamNamesTemp = listStreamsResult.getStreamNames();
                    streamNames.addAll(streamNamesTemp);
                }
                
                List<DataStreamVH> dataStreamList = new ArrayList<>();
                for(String streamName : streamNames) {
                    StreamDescription streamDescription = amazonKinesis.describeStream(new DescribeStreamRequest().withStreamName(streamName)).getStreamDescription();
                    ListTagsForStreamResult listTagsForStreamResult = amazonKinesis.listTagsForStream(new ListTagsForStreamRequest().withStreamName(streamName));
                    List<com.amazonaws.services.kinesis.model.Tag> tagsTemp = listTagsForStreamResult.getTags();
                    List<com.amazonaws.services.kinesis.model.Tag> tags = new ArrayList<>(tagsTemp);
                    while (listTagsForStreamResult.isHasMoreTags() && !tagsTemp.isEmpty()) {
                        listTagsForStreamResult = amazonKinesis.listTagsForStream(new ListTagsForStreamRequest().withExclusiveStartTagKey(tagsTemp.get(tagsTemp.size() - 1).getKey()));
                        tagsTemp = listTagsForStreamResult.getTags();
                        tags.addAll(tagsTemp);
                    }
                    dataStreamList.add(new DataStreamVH(streamDescription, tags));
                }
                if( !dataStreamList.isEmpty() ) {
                    log.debug(InventoryConstants.ACCOUNT + accountId +" Type : datastream "+region.getName() + " >> "+dataStreamList.size());
                    dataStream.put(accountId+delimiter+accountName+delimiter+region.getName(),dataStreamList);
                }
            }
        } catch(Exception e){
            log.warn(expPrefix+ region.getName()+InventoryConstants.ERROR_CAUSE +e.getMessage()+"\"}");
            ErrorManageUtil.uploadError(accountId, region.getName(),"datastream",e.getMessage());
        }
    }
    return dataStream;
}
 
Example #16
Source File: WatermarkTracker.java    From flink-stream-processing-refarch with Apache License 2.0 4 votes vote down vote up
public WatermarkTracker(String region, String streamName) {
  this.streamName = streamName;
  this.kinesisClient = AmazonKinesisClientBuilder.standard().withRegion(region).build();
}