com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration Java Examples

The following examples show how to use com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SampleKPLProducer.java    From kinesis-aggregation with Apache License 2.0 5 votes vote down vote up
/**
 * Given an AWS region, generate a configured Kinesis Producer.
 */
private static KinesisProducer getKinesisProducer(String region)
{
    KinesisProducerConfiguration config = new KinesisProducerConfiguration();
    config.setRegion(region);
    config.setMaxConnections(1);
    config.setRequestTimeout(60000);
    config.setRecordMaxBufferedTime(15000);
    config.setAggregationEnabled(true);

    return new KinesisProducer(config);
}
 
Example #2
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 
Example #3
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 
Example #4
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 
Example #5
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 
Example #6
Source File: KinesisConfigUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 
Example #7
Source File: KinesisEventProducer.java    From koupler with MIT License 5 votes vote down vote up
public KinesisEventProducer(String format, CommandLine cmd,
                            String propertiesFile, String streamName,
                            int throttleQueueSize, String appName) {
    this(throttleQueueSize);
    KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(propertiesFile);
    this.streamName = streamName;
    this.producer = new KinesisProducer(config);
    this.metrics = new KouplerMetrics(this, config, appName);
    this.format = FormatFactory.getFormat(format, cmd);
}
 
Example #8
Source File: KinesisEventConsumer.java    From koupler with MIT License 5 votes vote down vote up
public KinesisEventConsumer(String propertiesFile, String streamName, String appName, String initialPosition) {
    KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(propertiesFile);

    InitialPositionInStream position = InitialPositionInStream.valueOf(initialPosition);
    
    KinesisClientLibConfiguration clientConfig = new KinesisClientLibConfiguration(appName, streamName,
            new DefaultAWSCredentialsProviderChain(), appName)
                    .withRegionName(config.getRegion())
                    .withInitialPositionInStream(position);
    
    this.builder = new Worker.Builder().recordProcessorFactory(this).config(clientConfig);
}
 
Example #9
Source File: KinesisSink.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception {
    kinesisSinkConfig = KinesisSinkConfig.load(config);
    this.sinkContext = sinkContext;

    checkArgument(isNotBlank(kinesisSinkConfig.getAwsKinesisStreamName()), "empty kinesis-stream name");
    checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) || 
                  isNotBlank(kinesisSinkConfig.getAwsRegion()), 
                  "Either the aws-end-point or aws-region must be set");
    checkArgument(isNotBlank(kinesisSinkConfig.getAwsCredentialPluginParam()), "empty aws-credential param");

    KinesisProducerConfiguration kinesisConfig = new KinesisProducerConfiguration();
    kinesisConfig.setKinesisEndpoint(kinesisSinkConfig.getAwsEndpoint());
    kinesisConfig.setRegion(kinesisSinkConfig.getAwsRegion());
    kinesisConfig.setThreadingModel(ThreadingModel.POOLED);
    kinesisConfig.setThreadPoolSize(4);
    kinesisConfig.setCollectionMaxCount(1);
    AWSCredentialsProvider credentialsProvider = createCredentialProvider(
            kinesisSinkConfig.getAwsCredentialPluginName(),
            kinesisSinkConfig.getAwsCredentialPluginParam())
        .getCredentialProvider();
    kinesisConfig.setCredentialsProvider(credentialsProvider);

    this.streamName = kinesisSinkConfig.getAwsKinesisStreamName();
    this.kinesisProducer = new KinesisProducer(kinesisConfig);
    IS_PUBLISH_FAILED.set(this, FALSE);

    LOG.info("Kinesis sink started. {}", (ReflectionToStringBuilder.toString(kinesisConfig, ToStringStyle.SHORT_PREFIX_STYLE)));
}
 
Example #10
Source File: SampleProducer.java    From real-time-analytics-spark-streaming with Apache License 2.0 5 votes vote down vote up
/**
 * Here'll walk through some of the config options and create an instance of
 * KinesisProducer, which will be used to put records.
 *
 * @param region The region of the Kinesis stream being used.
 *
 * @return KinesisProducer instance used to put records.
 */
private static KinesisProducer getKinesisProducer(final String region) {
    KinesisProducerConfiguration config = new KinesisProducerConfiguration();
    config.setRegion(region);
    config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
    config.setMaxConnections(1);
    config.setRequestTimeout(60000);
    config.setRecordMaxBufferedTime(2000);
    config.setAggregationEnabled(false);

    KinesisProducer producer = new KinesisProducer(config);
    return producer;
}
 
Example #11
Source File: AmazonKinesisSinkTask.java    From kinesis-kafka-connector with Apache License 2.0 5 votes vote down vote up
private KinesisProducer getKinesisProducer() {
	KinesisProducerConfiguration config = new KinesisProducerConfiguration();
	config.setRegion(regionName);
	config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain());
	config.setMaxConnections(maxConnections);

	config.setAggregationEnabled(aggregration);

	// Limits the maximum allowed put rate for a shard, as a percentage of
	// the
	// backend limits.
	config.setRateLimit(rateLimit);

	// Maximum amount of time (milliseconds) a record may spend being
	// buffered
	// before it gets sent. Records may be sent sooner than this depending
	// on the
	// other buffering limits
	config.setRecordMaxBufferedTime(maxBufferedTime);

	// Set a time-to-live on records (milliseconds). Records that do not get
	// successfully put within the limit are failed.
	config.setRecordTtl(ttl);

	// Controls the number of metrics that are uploaded to CloudWatch.
	// Expected pattern: none|summary|detailed
	config.setMetricsLevel(metricsLevel);

	// Controls the granularity of metrics that are uploaded to CloudWatch.
	// Greater granularity produces more metrics.
	// Expected pattern: global|stream|shard
	config.setMetricsGranularity(metricsGranuality);

	// The namespace to upload metrics under.
	config.setMetricsNamespace(metricsNameSpace);

	return new KinesisProducer(config);

}
 
Example #12
Source File: KinesisPublisher.java    From james with Apache License 2.0 5 votes vote down vote up
private KinesisProducer createKinesisProducer(KinesisPublisherConfiguration configuration) {
    ProducerConfiguration pc = configuration.getProducer();
    KinesisProducerConfiguration kc = new KinesisProducerConfiguration();

    pc.isAggregationEnabled().ifPresent(kc::setAggregationEnabled);
    pc.getAggregationMaxCount().ifPresent(kc::setAggregationMaxCount);
    pc.getAggregationMaxSize().ifPresent(kc::setAggregationMaxSize);
    pc.getCloudwatchEndpoint().ifPresent(kc::setCloudwatchEndpoint);
    pc.getCloudwatchPort().ifPresent(kc::setCloudwatchPort);
    pc.getCollectionMaxCount().ifPresent(kc::setCollectionMaxCount);
    pc.getCollectionMaxSize().ifPresent(kc::setCollectionMaxSize);
    pc.getConnectTimeout().ifPresent(kc::setConnectTimeout);
    pc.isEnableCoreDumps().ifPresent(kc::setEnableCoreDumps);
    pc.isFailIfThrottled().ifPresent(kc::setFailIfThrottled);
    pc.getKinesisEndpoint().ifPresent(kc::setKinesisEndpoint);
    pc.getKinesisPort().ifPresent(kc::setKinesisPort);
    pc.getLogLevel().ifPresent(kc::setLogLevel);
    pc.getMaxConnections().ifPresent(kc::setMaxConnections);
    pc.getMinConnections().ifPresent(kc::setMinConnections);
    pc.getNativeExecutable().ifPresent(kc::setNativeExecutable);
    pc.getRateLimit().ifPresent(kc::setRateLimit);
    pc.getRecordMaxBufferedTime().ifPresent(kc::setRecordMaxBufferedTime);
    pc.getRecordTtl().ifPresent(kc::setRecordTtl);
    pc.getRegion().ifPresent(kc::setRegion);
    pc.getRequestTimeout().ifPresent(kc::setRequestTimeout);
    pc.getTempDirectory().ifPresent(kc::setTempDirectory);
    pc.isVerifyCertificate().ifPresent(kc::setVerifyCertificate);

    try {
        return new KinesisProducer(kc);
    } catch (Exception e) {
        LOG.error("Failed to initialize " + KinesisPublisher.class.getSimpleName(), e);
        return null;
    }
}
 
Example #13
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 
Example #14
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 
Example #15
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 
Example #16
Source File: KinesisConfigUtilTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 
Example #17
Source File: KinesisConfigUtil.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 
Example #18
Source File: KinesisConfigUtil.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Validate configuration properties for {@link FlinkKinesisProducer},
 * and return a constructed KinesisProducerConfiguration.
 */
public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) {
	checkNotNull(config, "config can not be null");

	validateAwsConfiguration(config);

	if (!config.containsKey(AWSConfigConstants.AWS_REGION)) {
		// per requirement in Amazon Kinesis Producer Library
		throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION));
	}

	KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config);
	kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION));

	kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config));

	// we explicitly lower the credential refresh delay (default is 5 seconds)
	// to avoid an ignorable interruption warning that occurs when shutting down the
	// KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10.
	kpc.setCredentialsRefreshDelay(100);

	// Override default values if they aren't specified by users
	if (!config.containsKey(RATE_LIMIT)) {
		kpc.setRateLimit(DEFAULT_RATE_LIMIT);
	}
	if (!config.containsKey(THREADING_MODEL)) {
		kpc.setThreadingModel(DEFAULT_THREADING_MODEL);
	}
	if (!config.containsKey(THREAD_POOL_SIZE)) {
		kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE);
	}

	return kpc;
}
 
Example #19
Source File: KinesisConfigUtilTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testRateLimitInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(100, kpc.getRateLimit());

	testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(150, kpc.getRateLimit());
}
 
Example #20
Source File: KinesisConfigUtilTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadingModelInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel());

	testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel());
}
 
Example #21
Source File: KinesisConfigUtilTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectlySetRegionInProducerConfiguration() {
	String region = "us-east-1";
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, region);
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals("incorrect region", region, kpc.getRegion());
}
 
Example #22
Source File: KinesisConfigUtilTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testThreadPoolSizeInProducerConfiguration() {
	Properties testConfig = new Properties();
	testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1");
	KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(10, kpc.getThreadPoolSize());

	testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12");
	kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig);

	assertEquals(12, kpc.getThreadPoolSize());
}
 
Example #23
Source File: FlinkKinesisProducer.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a {@link KinesisProducer}.
 * Exposed so that tests can inject mock producers easily.
 */
@VisibleForTesting
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return new KinesisProducer(producerConfig);
}
 
Example #24
Source File: FlinkKinesisProducerTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return mockProducer;
}
 
Example #25
Source File: FlinkKinesisProducerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return mockProducer;
}
 
Example #26
Source File: FlinkKinesisProducer.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a {@link KinesisProducer}.
 * Exposed so that tests can inject mock producers easily.
 */
@VisibleForTesting
protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) {
	return new KinesisProducer(producerConfig);
}
 
Example #27
Source File: FlinkKinesisProducer.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	schema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user"));

	// check and pass the configuration properties
	KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps);

	producer = getKinesisProducer(producerConfig);

	final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP);
	this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES);
	kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount);

	backpressureLatch = new TimeoutLatch();
	callback = new FutureCallback<UserRecordResult>() {
		@Override
		public void onSuccess(UserRecordResult result) {
			backpressureLatch.trigger();
			if (!result.isSuccessful()) {
				if (failOnError) {
					// only remember the first thrown exception
					if (thrownException == null) {
						thrownException = new RuntimeException("Record was not sent successful");
					}
				} else {
					LOG.warn("Record was not sent successful");
				}
			}
		}

		@Override
		public void onFailure(Throwable t) {
			backpressureLatch.trigger();
			if (failOnError) {
				thrownException = t;
			} else {
				LOG.warn("An exception occurred while processing a record", t);
			}
		}
	};

	if (this.customPartitioner != null) {
		this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks());
	}

	LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion());
}
 
Example #28
Source File: KPLClickEventsToKinesis.java    From aws-big-data-blog with Apache License 2.0 4 votes vote down vote up
public KPLClickEventsToKinesis(BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new KinesisProducer(new KinesisProducerConfiguration()
            .setRegion(REGION)
            .setRecordMaxBufferedTime(5000));
}
 
Example #29
Source File: AdvancedKPLClickEventsToKinesis.java    From aws-big-data-blog with Apache License 2.0 4 votes vote down vote up
protected AdvancedKPLClickEventsToKinesis(
        BlockingQueue<ClickEvent> inputQueue) {
    super(inputQueue);
    kinesis = new KinesisProducer(new KinesisProducerConfiguration()
            .setRegion(REGION));
}
 
Example #30
Source File: TalendKinesisProvider.java    From components with Apache License 2.0 4 votes vote down vote up
@Override
public IKinesisProducer createKinesisProducer(KinesisProducerConfiguration config) {
    // TODO implement when support Kinesis output, as it only used for write
    return null;
}