com.amazonaws.services.dynamodbv2.AmazonDynamoDBStreams Java Examples

public DynamoDBTableReplicator(
		AmazonDynamoDB dynamoDBClient,
		AmazonDynamoDBStreams streamsClient,
		AWSCredentialsProvider awsCredentialsProvider,
		ExecutorService executorService,
		TableEmitter emitter,
		String tableName) throws SQLException {
	this.dynamoDBClient = dynamoDBClient;
	this.streamsClient = streamsClient;
	this.awsCredentialsProvider = awsCredentialsProvider;
	this.executor = executorService;
	this.emitter = emitter;
	this.dynamoTableName = tableName;
	this.addColumnsEnabled = true;
	this.useCitus = false;
	this.useLowerCaseColumnNames = false;
	this.tableSchema = emitter.fetchSchema(this.dynamoTableName);
}
 

public KinesisMessageChannelBinder(KinesisBinderConfigurationProperties configurationProperties,
		KinesisStreamProvisioner provisioningProvider, AmazonKinesisAsync amazonKinesis,
		AWSCredentialsProvider awsCredentialsProvider,
		@Nullable AmazonDynamoDB dynamoDBClient,
		@Nullable AmazonDynamoDBStreams dynamoDBStreams,
		@Nullable AmazonCloudWatch cloudWatchClient) {

	super(headersToMap(configurationProperties), provisioningProvider);
	Assert.notNull(amazonKinesis, "'amazonKinesis' must not be null");
	Assert.notNull(awsCredentialsProvider, "'awsCredentialsProvider' must not be null");
	this.configurationProperties = configurationProperties;
	this.amazonKinesis = amazonKinesis;
	this.cloudWatchClient = cloudWatchClient;
	this.dynamoDBClient = dynamoDBClient;
	this.awsCredentialsProvider = awsCredentialsProvider;

	if (dynamoDBStreams != null) {
		this.dynamoDBStreamsAdapter = new AmazonDynamoDBStreamsAdapterClient(dynamoDBStreams);
	}
	else {
		this.dynamoDBStreamsAdapter = null;
	}
}
 

@Override
public AmazonDynamoDBStreams amazonDynamoDBStreams() {
    return decorateWithConfigsAndBuild(
        AmazonDynamoDBStreamsClientBuilder.standard(),
        LocalstackDocker::getEndpointDynamoDBStreams
    );
}
 

@Bean
@ConditionalOnMissingBean
public AmazonDynamoDBStreams dynamoDBStreams() {
	if (this.hasInputs) {
		return AmazonDynamoDBStreamsClientBuilder.standard()
				.withCredentials(this.awsCredentialsProvider)
				.withRegion(this.region)
				.build();
	}
	else {
		return null;
	}
}
 

public AmazonDynamoDBStreams buildDynamoDBStreamsClient(AwsCredentialProviderPlugin credPlugin) {
    AmazonDynamoDBStreamsClientBuilder builder = AmazonDynamoDBStreamsClientBuilder.standard();

    if (!this.getAwsEndpoint().isEmpty()) {
        builder.setEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(this.getAwsEndpoint(), this.getAwsRegion()));
    }
    if (!this.getAwsRegion().isEmpty()) {
        builder.setRegion(this.getAwsRegion());
    }
    builder.setCredentials(credPlugin.getCredentialProvider());
    return builder.build();
}
 

/**
 * Check whether Streams is enabled on the given argument with the given stream view type
 * 
 * @param streamsClient 
 *            streams client used to access the given stream
 * @param streamArn
 *            the stream ARN to check against
 * @param viewType
 *            the stream view type to check against
 * @return a boolean indicating whether the given stream is enabled and matches the given stream view type
 */
public static boolean isStreamsEnabled(AmazonDynamoDBStreams streamsClient, String streamArn,
                                       StreamViewType viewType) {
    // Get and check stream description
    StreamDescription result = streamsClient.describeStream(new DescribeStreamRequest().withStreamArn(streamArn))
        .getStreamDescription();
    if (result.getStreamStatus().equalsIgnoreCase(DynamoDBConnectorConstants.ENABLED_STRING)
        && result.getStreamViewType().equalsIgnoreCase(viewType.toString())) {
        return true;
    }
    log.error(DynamoDBConnectorConstants.STREAM_NOT_READY + " StreamARN: " + streamArn);
    return false;
}
 

@Bean
public AmazonDynamoDBStreams amazonDynamoDBStreams() {
    return amazonClientsHolder.amazonDynamoDBStreams();
}
 

@Override
    public void open(Map<String, Object> config, SourceContext sourceContext) throws Exception {
        this.dynamodbSourceConfig = DynamoDBSourceConfig.load(config);
        
        checkArgument(isNotBlank(dynamodbSourceConfig.getAwsDynamodbStreamArn()), "empty dynamo-stream arn");
//       Even if the endpoint is set, it seems to require a region to go with it
        checkArgument(isNotBlank(dynamodbSourceConfig.getAwsRegion()),
                     "The aws-region must be set");
        checkArgument(isNotBlank(dynamodbSourceConfig.getAwsCredentialPluginParam()), "empty aws-credential param");
        
        if (dynamodbSourceConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
            checkArgument((dynamodbSourceConfig.getStartAtTime() != null),"Timestamp must be specified");
        }
        
        queue = new LinkedBlockingQueue<> (dynamodbSourceConfig.getReceiveQueueSize());
        workerId = InetAddress.getLocalHost().getCanonicalHostName() + ":" + UUID.randomUUID();
        
        AwsCredentialProviderPlugin credentialsProvider = createCredentialProvider(
                dynamodbSourceConfig.getAwsCredentialPluginName(),
                dynamodbSourceConfig.getAwsCredentialPluginParam());

        AmazonDynamoDBStreams dynamoDBStreamsClient = dynamodbSourceConfig.buildDynamoDBStreamsClient(credentialsProvider);
        AmazonDynamoDBStreamsAdapterClient adapterClient = new AmazonDynamoDBStreamsAdapterClient(dynamoDBStreamsClient);
        recordProcessorFactory = new StreamsRecordProcessorFactory(queue, dynamodbSourceConfig);

        kinesisClientLibConfig = new KinesisClientLibConfiguration(dynamodbSourceConfig.getApplicationName(),
                dynamodbSourceConfig.getAwsDynamodbStreamArn(),
                credentialsProvider.getCredentialProvider(),
                workerId)
                .withRegionName(dynamodbSourceConfig.getAwsRegion())
                .withInitialPositionInStream(dynamodbSourceConfig.getInitialPositionInStream());

        if(kinesisClientLibConfig.getInitialPositionInStream() == InitialPositionInStream.AT_TIMESTAMP) {
            kinesisClientLibConfig.withTimestampAtInitialPositionInStream(dynamodbSourceConfig.getStartAtTime());
        }

        worker = StreamsWorkerFactory.createDynamoDbStreamsWorker(recordProcessorFactory,
                kinesisClientLibConfig,
                adapterClient,
                dynamodbSourceConfig.buildDynamoDBClient(credentialsProvider),
                dynamodbSourceConfig.buildCloudwatchClient(credentialsProvider));

        workerThread = new Thread(worker);
        workerThread.setDaemon(true);
        threadEx = null;
        workerThread.setUncaughtExceptionHandler((t, ex) -> {
            threadEx = ex;
            log.error("Worker died with error", ex);
        });
        workerThread.start();
    }
 

public Worker createWorker() {

        // use default credential provider chain to locate appropriate credentials
        final AWSCredentialsProvider credentialsProvider = new DefaultAWSCredentialsProviderChain();

        // initialize DynamoDB client and set the endpoint properly for source table / region
        final AmazonDynamoDB dynamodbClient = AmazonDynamoDBClientBuilder.standard()
                .withCredentials(credentialsProvider)
                .withEndpointConfiguration(createEndpointConfiguration(sourceRegion, sourceDynamodbEndpoint, AmazonDynamoDB.ENDPOINT_PREFIX))
                .build();

        // initialize Streams client
        final AwsClientBuilder.EndpointConfiguration streamsEndpointConfiguration = createEndpointConfiguration(sourceRegion,
                sourceDynamodbStreamsEndpoint, AmazonDynamoDBStreams.ENDPOINT_PREFIX);
        final ClientConfiguration streamsClientConfig = new ClientConfiguration().withGzip(false);
        final AmazonDynamoDBStreams streamsClient = AmazonDynamoDBStreamsClientBuilder.standard()
                .withCredentials(credentialsProvider)
                .withEndpointConfiguration(streamsEndpointConfiguration)
                .withClientConfiguration(streamsClientConfig)
                .build();

        // obtain the Stream ID associated with the source table
        final String streamArn = dynamodbClient.describeTable(sourceTable).getTable().getLatestStreamArn();
        final boolean streamEnabled = DynamoDBConnectorUtilities.isStreamsEnabled(streamsClient, streamArn, DynamoDBConnectorConstants.NEW_AND_OLD);
        Preconditions.checkArgument(streamArn != null, DynamoDBConnectorConstants.MSG_NO_STREAMS_FOUND);
        Preconditions.checkState(streamEnabled, DynamoDBConnectorConstants.STREAM_NOT_READY);

        // initialize DynamoDB client for KCL
        final AmazonDynamoDB kclDynamoDBClient = AmazonDynamoDBClientBuilder.standard()
                .withCredentials(credentialsProvider)
                .withEndpointConfiguration(createKclDynamoDbEndpointConfiguration())
                .build();

        // initialize DynamoDB Streams Adapter client and set the Streams endpoint properly
        final AmazonDynamoDBStreamsAdapterClient streamsAdapterClient = new AmazonDynamoDBStreamsAdapterClient(streamsClient);

        // initialize CloudWatch client and set the region to emit metrics to
        final AmazonCloudWatch kclCloudWatchClient;
        if (isPublishCloudWatch) {
            kclCloudWatchClient = AmazonCloudWatchClientBuilder.standard()
                    .withCredentials(credentialsProvider)
                    .withRegion(kclRegion.or(sourceRegion).getName()).build();
        } else {
            kclCloudWatchClient = new NoopCloudWatch();
        }

        // try to get taskname from command line arguments, auto generate one if needed
        final AwsClientBuilder.EndpointConfiguration destinationEndpointConfiguration = createEndpointConfiguration(destinationRegion,
                destinationDynamodbEndpoint, AmazonDynamoDB.ENDPOINT_PREFIX);
        final String actualTaskName = DynamoDBConnectorUtilities.getTaskName(sourceRegion, destinationRegion, taskName, sourceTable, destinationTable);

        // set the appropriate Connector properties for the destination KCL configuration
        final Properties properties = new Properties();
        properties.put(DynamoDBStreamsConnectorConfiguration.PROP_APP_NAME, actualTaskName);
        properties.put(DynamoDBStreamsConnectorConfiguration.PROP_DYNAMODB_ENDPOINT, destinationEndpointConfiguration.getServiceEndpoint());
        properties.put(DynamoDBStreamsConnectorConfiguration.PROP_DYNAMODB_DATA_TABLE_NAME, destinationTable);
        properties.put(DynamoDBStreamsConnectorConfiguration.PROP_REGION_NAME, destinationRegion.getName());

        // create the record processor factory based on given pipeline and connector configurations
        // use the master to replicas pipeline
        final KinesisConnectorRecordProcessorFactory<Record, Record> factory = new KinesisConnectorRecordProcessorFactory<>(
                new DynamoDBMasterToReplicasPipeline(), new DynamoDBStreamsConnectorConfiguration(properties, credentialsProvider));

        // create the KCL configuration with default values
        final KinesisClientLibConfiguration kclConfig = new KinesisClientLibConfiguration(actualTaskName,
                streamArn,
                credentialsProvider,
                DynamoDBConnectorConstants.WORKER_LABEL + actualTaskName + UUID.randomUUID().toString())
                // worker will use checkpoint table if available, otherwise it is safer
                // to start at beginning of the stream
                .withInitialPositionInStream(InitialPositionInStream.TRIM_HORIZON)
                // we want the maximum batch size to avoid network transfer latency overhead
                .withMaxRecords(getRecordsLimit.or(DynamoDBConnectorConstants.STREAMS_RECORDS_LIMIT))
                // wait a reasonable amount of time - default 0.5 seconds
                .withIdleTimeBetweenReadsInMillis(DynamoDBConnectorConstants.IDLE_TIME_BETWEEN_READS)
                // Remove calls to GetShardIterator
                .withValidateSequenceNumberBeforeCheckpointing(false)
                // make parent shard poll interval tunable to decrease time to run integration test
                .withParentShardPollIntervalMillis(parentShardPollIntervalMillis.or(DynamoDBConnectorConstants.DEFAULT_PARENT_SHARD_POLL_INTERVAL_MILLIS))
                // avoid losing leases too often - default 60 seconds
                .withFailoverTimeMillis(DynamoDBConnectorConstants.KCL_FAILOVER_TIME);

        // create the KCL worker for this connector
        return new Worker(factory, kclConfig, streamsAdapterClient, kclDynamoDBClient, kclCloudWatchClient);
    }
 

AmazonDynamoDBStreams amazonDynamoDBStreams();