com.amazonaws.services.kinesis.connectors.interfaces.IEmitter Java Examples

@SuppressWarnings("deprecation")
@Test
public void insertTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_INSERT, ITEM1_INSERT.getDynamodb().getSizeBytes().intValue(), ITEM1_INSERT.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_INSERT.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM1_INSERT.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(1, buffered.size());
    assertTrue(buffered.contains(ITEM1_INSERT));

    // Emit record
    resetAll(DYNAMODB);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.putItemAsync(EasyMock.anyObject(PutItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER);

    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@SuppressWarnings("deprecation")
@Test
public void modifyTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_MODIFY, ITEM1_MODIFY.getDynamodb().getSizeBytes().intValue(), ITEM1_MODIFY.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_MODIFY.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM1_MODIFY.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(1, buffered.size());
    assertTrue(buffered.contains(ITEM1_MODIFY));

    // Emit record
    resetAll(DYNAMODB);
    DYNAMODB.putItemAsync(EasyMock.anyObject(PutItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@SuppressWarnings("deprecation")
@Test
public void removeTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_REMOVE, ITEM1_REMOVE.getDynamodb().getSizeBytes().intValue(), ITEM1_REMOVE.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_REMOVE.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM1_REMOVE.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(1, buffered.size());
    assertTrue(buffered.contains(ITEM1_REMOVE));

    // Emit record
    resetAll(DYNAMODB);
    DYNAMODB.deleteItemAsync(anyObject(DeleteItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

@SuppressWarnings("deprecation")
@Test
public void multipleRecordsEmitsTest() throws Exception {
    // Set up the buffer and do sanity checks
    buffer.clear();
    buffer.consumeRecord(ITEM1_INSERT, ITEM1_INSERT.getDynamodb().getSizeBytes().intValue(), ITEM1_INSERT.getDynamodb().getSequenceNumber());
    buffer.consumeRecord(ITEM2_INSERT, ITEM2_INSERT.getDynamodb().getSizeBytes().intValue(), ITEM2_INSERT.getDynamodb().getSequenceNumber());
    assertEquals(ITEM1_INSERT.getDynamodb().getSequenceNumber(), buffer.getFirstSequenceNumber());
    assertEquals(ITEM2_INSERT.getDynamodb().getSequenceNumber(), buffer.getLastSequenceNumber());
    List<Record> buffered = buffer.getRecords();
    assertEquals(2, buffered.size());
    assertTrue(buffered.contains(ITEM1_INSERT));
    assertTrue(buffered.contains(ITEM2_INSERT));

    // Emit record
    resetAll(DYNAMODB);
    DYNAMODB.putItemAsync(EasyMock.anyObject(PutItemRequest.class), anyObject(AsyncHandler.class));
    expectLastCall().andAnswer(SUCCESS_ANSWER).times(2);
    expectNew(AmazonDynamoDBAsyncClient.class, new Class<?>[] {AWSCredentialsProvider.class, ClientConfiguration.class, ExecutorService.class}, anyObject(AWSCredentialsProvider.class), anyObject(ClientConfiguration.class), anyObject(ExecutorService.class)).andReturn(DYNAMODB);
    DYNAMODB.setEndpoint(EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    replayAll(DYNAMODB);
    IEmitter<Record> instance = createEmitterInstance();
    assertTrue(instance.emit(new UnmodifiableBuffer<Record>(buffer)).isEmpty());
    verifyAll();
}
 

public KinesisConnectorRecordProcessor(IBuffer<T> buffer,
        IFilter<T> filter,
        IEmitter<U> emitter,
        ITransformerBase<T, U> transformer,
        KinesisConnectorConfiguration configuration) {
    if (buffer == null || filter == null || emitter == null || transformer == null) {
        throw new IllegalArgumentException("buffer, filter, emitter, and transformer must not be null");
    }
    this.buffer = buffer;
    this.filter = filter;
    this.emitter = emitter;
    this.transformer = transformer;
    // Limit must be greater than zero
    if (configuration.RETRY_LIMIT <= 0) {
        retryLimit = 1;
    } else {
        retryLimit = configuration.RETRY_LIMIT;
    }
    this.backoffInterval = configuration.BACKOFF_INTERVAL;
}
 

@Before
@SuppressWarnings("unchecked")
public void setUp() {
    // control object used to create mock dependencies
    control = EasyMock.createControl();

    // mock dependencies
    emitter = control.createMock(IEmitter.class);
    transformer = control.createMock(ITransformer.class);
    buffer = control.createMock(IBuffer.class);
    filter = control.createMock(IFilter.class);
    checkpointer = control.createMock(IRecordProcessorCheckpointer.class);
    // use a real configuration to get actual default values (not anything created by EasyMock)
    configuration = new KinesisConnectorConfiguration(new Properties(), new DefaultAWSCredentialsProviderChain());

}
 

@Override
public IEmitter<Record> getEmitter(final KinesisConnectorConfiguration configuration) {
    if (configuration instanceof DynamoDBStreamsConnectorConfiguration) {
        return new DynamoDBReplicationEmitter((DynamoDBStreamsConnectorConfiguration) configuration);
    } else {
        throw new IllegalArgumentException(this + " needs a DynamoDBStreamsConnectorConfiguration argument.");
    }

}
 

@Override
protected IEmitter<Record> createEmitterInstance() {
    Properties properties = new Properties();
    properties.setProperty("APP_NAME", "TEST");
    properties.setProperty("DYNAMODB_ENDPOINT", "ENDPOINT");
    properties.setProperty("REGION_NAME", "REGION");
    properties.setProperty("DYNAMODB_DATA_TABLE_NAME", "TABLE");
    AWSStaticCredentialsProvider credentialProvider = new AWSStaticCredentialsProvider(new BasicAWSCredentials("Access", "Secret"));
    return new DynamoDBReplicationEmitter(new DynamoDBStreamsConnectorConfiguration(properties, credentialProvider),
        new AmazonDynamoDBAsyncClient(credentialProvider, new ClientConfiguration().withMaxErrorRetry(0), Executors.newFixedThreadPool(MAX_THREADS)), null);
}
 

@Override
public IRecordProcessor createProcessor() {
    try {
        IBuffer<T> buffer = pipeline.getBuffer(configuration);
        IEmitter<U> emitter = pipeline.getEmitter(configuration);
        ITransformerBase<T, U> transformer = pipeline.getTransformer(configuration);
        IFilter<T> filter = pipeline.getFilter(configuration);
        KinesisConnectorRecordProcessor<T, U> processor =
                new KinesisConnectorRecordProcessor<T, U>(buffer, filter, emitter, transformer, configuration);
        return processor;
    } catch (Throwable t) {
        throw new RuntimeException(t);
    }
}
 

@Override
public IEmitter<byte[]> getEmitter(KinesisConnectorConfiguration configuration) {
    return new S3Emitter(configuration);
}
 

@Override
public IEmitter<ElasticsearchObject> getEmitter(KinesisConnectorConfiguration configuration) {
    return new ElasticsearchEmitter(configuration);
}
 

@Override
public IEmitter<Map<String, AttributeValue>> getEmitter(KinesisConnectorConfiguration configuration) {
    return new DynamoDBEmitter(configuration);
}
 

@Override
public IEmitter<byte[]> getEmitter(KinesisConnectorConfiguration configuration) {
    return new RedshiftBasicEmitter(configuration);
}
 

@Override
public IEmitter<String> getEmitter(KinesisConnectorConfiguration configuration) {
    return new RedshiftManifestEmitter(configuration);
}
 

@Override
public IEmitter<byte[]> getEmitter(KinesisConnectorConfiguration configuration) {
    return new S3ManifestEmitter(configuration);
}
 

@Override
public IEmitter<Map<String,String>> getEmitter(KinesisConnectorConfiguration configuration) {
    return new HBaseEmitter((EMRHBaseKinesisConnectorConfiguration) configuration);
}
 

@Override
protected IEmitter<Record> createEmitterInstance() {
    return new DynamoDBReplicationEmitter("TEST", "ENDPOINT", "REGION", "TABLE", null, new AWSStaticCredentialsProvider(new BasicAWSCredentials("Access", "Secret")));
}
 

protected abstract IEmitter<Record> createEmitterInstance();