com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration Java Examples

/**
 * Given an AWS region, generate a configured Kinesis Producer.
 */
private static KinesisProducer getKinesisProducer(String region)
{
    KinesisProducerConfiguration config = new KinesisProducerConfiguration();
    config.setRegion(region);
    config.setMaxConnections(1);
    config.setRequestTimeout(60000);
    config.setRecordMaxBufferedTime(15000);
    config.setAggregationEnabled(true);

    return new KinesisProducer(config);
}
 

@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 

@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 

@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 

@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 

public KinesisEventProducer(String format, CommandLine cmd,
                            String propertiesFile, String streamName,
                            int throttleQueueSize, String appName) {
    this(throttleQueueSize);
    KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(propertiesFile);
    this.streamName = streamName;
    this.producer = new KinesisProducer(config);
    this.metrics = new KouplerMetrics(this, config, appName);
    this.format = FormatFactory.getFormat(format, cmd);
}
 

public KinesisEventConsumer(String propertiesFile, String streamName, String appName, String initialPosition) {
    KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(propertiesFile);

    InitialPositionInStream position = InitialPositionInStream.valueOf(initialPosition);
    
    KinesisClientLibConfiguration clientConfig = new KinesisClientLibConfiguration(appName, streamName,
            new DefaultAWSCredentialsProviderChain(), appName)
                    .withRegionName(config.getRegion())
                    .withInitialPositionInStream(position);
    
    this.builder = new Worker.Builder().recordProcessorFactory(this).config(clientConfig);
}
 

@Override
public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception {
    kinesisSinkConfig = KinesisSinkConfig.load(config);
    this.sinkContext = sinkContext;

    checkArgument(isNotBlank(kinesisSinkConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
    checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) || 
                  isNotBlank(kinesisSinkConfig.getAwsRegion()), 
                  "Either the aws-end-point or aws-region must be set");
    checkArgument(isNotBlank(kinesisSinkConfig.getAwsCredentialPluginParam()), "empty aws-credential param");

    KinesisProducerConfiguration kinesisConfig = new KinesisProducerConfiguration();
    kinesisConfig.setKinesisEndpoint(kinesisSinkConfig.getAwsEndpoint());
    kinesisConfig.setRegion(kinesisSinkConfig.getAwsRegion());
    kinesisConfig.setThreadingModel(ThreadingModel.POOLED);
    kinesisConfig.setThreadPoolSize(4);
    kinesisConfig.setCollectionMaxCount(1);
    AWSCredentialsProvider credentialsProvider = createCredentialProvider(
            kinesisSinkConfig.getAwsCredentialPluginName(),
            kinesisSinkConfig.getAwsCredentialPluginParam())
        .getCredentialProvider();
    kinesisConfig.setCredentialsProvider(credentialsProvider);

    this.streamName = kinesisSinkConfig.getAwsKinesisStreamName();
    this.kinesisProducer = new KinesisProducer(kinesisConfig);
    IS_PUBLISH_FAILED.set(this, FALSE);

    LOG.info("Kinesis sink started. {}", (ReflectionToStringBuilder.toString(kinesisConfig, ToStringStyle.SHORT_PREFIX_STYLE)));
}
 

/**
 * Here'll walk through some of the config options and create an instance of
 * KinesisProducer, which will be used to put records.
 *
 * @param region The region of the Kinesis stream being used.
 *
 * @return KinesisProducer instance used to put records.
 */
private static KinesisProducer getKinesisProducer(final String region) {
    KinesisProducerConfiguration config = new KinesisProducerConfiguration();
    config.setRegion(region);
    config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
    config.setMaxConnections(1);
    config.setRequestTimeout(60000);
    config.setRecordMaxBufferedTime(2000);
    config.setAggregationEnabled(false);

    KinesisProducer producer = new KinesisProducer(config);
    return producer;
}
 

private KinesisProducer getKinesisProducer() {
	KinesisProducerConfiguration config = new KinesisProducerConfiguration();
	config.setRegion(regionName);
	config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
	config.setMaxConnections(maxConnections);

	config.setAggregationEnabled(aggregration);

	// Limits the maximum allowed put rate for a shard, as a percentage of
	// the
	// backend limits.
	config.setRateLimit(rateLimit);

	// Maximum amount of time (milliseconds) a record may spend being
	// buffered
	// before it gets sent. Records may be sent sooner than this depending
	// on the
	// other buffering limits
	config.setRecordMaxBufferedTime(maxBufferedTime);

	// Set a time-to-live on records (milliseconds). Records that do not get
	// successfully put within the limit are failed.
	config.setRecordTtl(ttl);

	// Controls the number of metrics that are uploaded to CloudWatch.
	// Expected pattern: none|summary|detailed
	config.setMetricsLevel(metricsLevel);

	// Controls the granularity of metrics that are uploaded to CloudWatch.
	// Greater granularity produces more metrics.
	// Expected pattern: global|stream|shard
	config.setMetricsGranularity(metricsGranuality);

	// The namespace to upload metrics under.
	config.setMetricsNamespace(metricsNameSpace);

	return new KinesisProducer(config);

}
 

private KinesisProducer createKinesisProducer(KinesisPublisherConfiguration configuration) {
    ProducerConfiguration pc = configuration.getProducer();
    KinesisProducerConfiguration kc = new KinesisProducerConfiguration();

    pc.isAggregationEnabled().ifPresent(kc::setAggregationEnabled);
    pc.getAggregationMaxCount().ifPresent(kc::setAggregationMaxCount);
    pc.getAggregationMaxSize().ifPresent(kc::setAggregationMaxSize);
    pc.getCloudwatchEndpoint().ifPresent(kc::setCloudwatchEndpoint);
    pc.getCloudwatchPort().ifPresent(kc::setCloudwatchPort);
    pc.getCollectionMaxCount().ifPresent(kc::setCollectionMaxCount);
    pc.getCollectionMaxSize().ifPresent(kc::setCollectionMaxSize);
    pc.getConnectTimeout().ifPresent(kc::setConnectTimeout);
    pc.isEnableCoreDumps().ifPresent(kc::setEnableCoreDumps);
    pc.isFailIfThrottled().ifPresent(kc::setFailIfThrottled);
    pc.getKinesisEndpoint().ifPresent(kc::setKinesisEndpoint);
    pc.getKinesisPort().ifPresent(kc::setKinesisPort);
    pc.getLogLevel().ifPresent(kc::setLogLevel);
    pc.getMaxConnections().ifPresent(kc::setMaxConnections);
    pc.getMinConnections().ifPresent(kc::setMinConnections);
    pc.getNativeExecutable().ifPresent(kc::setNativeExecutable);
    pc.getRateLimit().ifPresent(kc::setRateLimit);
    pc.getRecordMaxBufferedTime().ifPresent(kc::setRecordMaxBufferedTime);
    pc.getRecordTtl().ifPresent(kc::setRecordTtl);
    pc.getRegion().ifPresent(kc::setRegion);
    pc.getRequestTimeout().ifPresent(kc::setRequestTimeout);
    pc.getTempDirectory().ifPresent(kc::setTempDirectory);
    pc.isVerifyCertificate().ifPresent(kc::setVerifyCertificate);

    try {
        return new KinesisProducer(kc);
    } catch (Exception e) {
        LOG.error("Failed to initialize " + KinesisPublisher.class.getSimpleName(), e);
        return null;
    }
}
 

@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 

@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 

@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 

@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 

/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 

@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 

@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 

@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 

@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 

/**
 * Creates a {@link KinesisProducer}.
 * Exposed so that tests can inject mock producers easily.
 */
@VisibleForTesting
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return new KinesisProducer(producerConfig);
}
 

@Override
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return mockProducer;
}
 

@Override
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return mockProducer;
}
 

/**
 * Creates a {@link KinesisProducer}.
 * Exposed so that tests can inject mock producers easily.
 */
@VisibleForTesting
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return new KinesisProducer(producerConfig);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	schema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user"));

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}
 

public KPLClickEventsToKinesis(BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new KinesisProducer(new KinesisProducerConfiguration()
            .setRegion(REGION)
            .setRecordMaxBufferedTime(5000));
}
 

protected AdvancedKPLClickEventsToKinesis(
        BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new KinesisProducer(new KinesisProducerConfiguration()
            .setRegion(REGION));
}
 

@Override
public IKinesisProducer createKinesisProducer(KinesisProducerConfiguration config) {
    // TODO implement when support Kinesis output, as it only used for write
    return null;
}