com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration Java Examples
The following examples show how to use
com.amazonaws.services.kinesis.producer.KinesisProducerConfiguration.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SampleKPLProducer.java From kinesis-aggregation with Apache License 2.0 | 5 votes |
/** * Given an AWS region, generate a configured Kinesis Producer. */ private static KinesisProducer getKinesisProducer(String region) { KinesisProducerConfiguration config = new KinesisProducerConfiguration(); config.setRegion(region); config.setMaxConnections(1); config.setRequestTimeout(60000); config.setRecordMaxBufferedTime(15000); config.setAggregationEnabled(true); return new KinesisProducer(config); }
Example #2
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCorrectlySetRegionInProducerConfiguration() { String region = "us-east-1"; Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, region); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals("incorrect region", region, kpc.getRegion()); }
Example #3
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testThreadPoolSizeInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(10, kpc.getThreadPoolSize()); testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(12, kpc.getThreadPoolSize()); }
Example #4
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testThreadingModelInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel()); testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel()); }
Example #5
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testRateLimitInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(100, kpc.getRateLimit()); testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(150, kpc.getRateLimit()); }
Example #6
Source File: KinesisConfigUtil.java From flink with Apache License 2.0 | 5 votes |
/** * Validate configuration properties for {@link FlinkKinesisProducer}, * and return a constructed KinesisProducerConfiguration. */ public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) { checkNotNull(config, "config can not be null"); validateAwsConfiguration(config); if (!config.containsKey(AWSConfigConstants.AWS_REGION)) { // per requirement in Amazon Kinesis Producer Library throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION)); } KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config); kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION)); kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config)); // we explicitly lower the credential refresh delay (default is 5 seconds) // to avoid an ignorable interruption warning that occurs when shutting down the // KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10. kpc.setCredentialsRefreshDelay(100); // Override default values if they aren't specified by users if (!config.containsKey(RATE_LIMIT)) { kpc.setRateLimit(DEFAULT_RATE_LIMIT); } if (!config.containsKey(THREADING_MODEL)) { kpc.setThreadingModel(DEFAULT_THREADING_MODEL); } if (!config.containsKey(THREAD_POOL_SIZE)) { kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE); } return kpc; }
Example #7
Source File: KinesisEventProducer.java From koupler with MIT License | 5 votes |
public KinesisEventProducer(String format, CommandLine cmd, String propertiesFile, String streamName, int throttleQueueSize, String appName) { this(throttleQueueSize); KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(propertiesFile); this.streamName = streamName; this.producer = new KinesisProducer(config); this.metrics = new KouplerMetrics(this, config, appName); this.format = FormatFactory.getFormat(format, cmd); }
Example #8
Source File: KinesisEventConsumer.java From koupler with MIT License | 5 votes |
public KinesisEventConsumer(String propertiesFile, String streamName, String appName, String initialPosition) { KinesisProducerConfiguration config = KinesisProducerConfiguration.fromPropertiesFile(propertiesFile); InitialPositionInStream position = InitialPositionInStream.valueOf(initialPosition); KinesisClientLibConfiguration clientConfig = new KinesisClientLibConfiguration(appName, streamName, new DefaultAWSCredentialsProviderChain(), appName) .withRegionName(config.getRegion()) .withInitialPositionInStream(position); this.builder = new Worker.Builder().recordProcessorFactory(this).config(clientConfig); }
Example #9
Source File: KinesisSink.java From pulsar with Apache License 2.0 | 5 votes |
@Override public void open(Map<String, Object> config, SinkContext sinkContext) throws Exception { kinesisSinkConfig = KinesisSinkConfig.load(config); this.sinkContext = sinkContext; checkArgument(isNotBlank(kinesisSinkConfig.getAwsKinesisStreamName()), "empty kinesis-stream name"); checkArgument(isNotBlank(kinesisSinkConfig.getAwsEndpoint()) || isNotBlank(kinesisSinkConfig.getAwsRegion()), "Either the aws-end-point or aws-region must be set"); checkArgument(isNotBlank(kinesisSinkConfig.getAwsCredentialPluginParam()), "empty aws-credential param"); KinesisProducerConfiguration kinesisConfig = new KinesisProducerConfiguration(); kinesisConfig.setKinesisEndpoint(kinesisSinkConfig.getAwsEndpoint()); kinesisConfig.setRegion(kinesisSinkConfig.getAwsRegion()); kinesisConfig.setThreadingModel(ThreadingModel.POOLED); kinesisConfig.setThreadPoolSize(4); kinesisConfig.setCollectionMaxCount(1); AWSCredentialsProvider credentialsProvider = createCredentialProvider( kinesisSinkConfig.getAwsCredentialPluginName(), kinesisSinkConfig.getAwsCredentialPluginParam()) .getCredentialProvider(); kinesisConfig.setCredentialsProvider(credentialsProvider); this.streamName = kinesisSinkConfig.getAwsKinesisStreamName(); this.kinesisProducer = new KinesisProducer(kinesisConfig); IS_PUBLISH_FAILED.set(this, FALSE); LOG.info("Kinesis sink started. {}", (ReflectionToStringBuilder.toString(kinesisConfig, ToStringStyle.SHORT_PREFIX_STYLE))); }
Example #10
Source File: SampleProducer.java From real-time-analytics-spark-streaming with Apache License 2.0 | 5 votes |
/** * Here'll walk through some of the config options and create an instance of * KinesisProducer, which will be used to put records. * * @param region The region of the Kinesis stream being used. * * @return KinesisProducer instance used to put records. */ private static KinesisProducer getKinesisProducer(final String region) { KinesisProducerConfiguration config = new KinesisProducerConfiguration(); config.setRegion(region); config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain()); config.setMaxConnections(1); config.setRequestTimeout(60000); config.setRecordMaxBufferedTime(2000); config.setAggregationEnabled(false); KinesisProducer producer = new KinesisProducer(config); return producer; }
Example #11
Source File: AmazonKinesisSinkTask.java From kinesis-kafka-connector with Apache License 2.0 | 5 votes |
private KinesisProducer getKinesisProducer() { KinesisProducerConfiguration config = new KinesisProducerConfiguration(); config.setRegion(regionName); config.setCredentialsProvider(new DefaultAWSCredentialsProviderChain()); config.setMaxConnections(maxConnections); config.setAggregationEnabled(aggregration); // Limits the maximum allowed put rate for a shard, as a percentage of // the // backend limits. config.setRateLimit(rateLimit); // Maximum amount of time (milliseconds) a record may spend being // buffered // before it gets sent. Records may be sent sooner than this depending // on the // other buffering limits config.setRecordMaxBufferedTime(maxBufferedTime); // Set a time-to-live on records (milliseconds). Records that do not get // successfully put within the limit are failed. config.setRecordTtl(ttl); // Controls the number of metrics that are uploaded to CloudWatch. // Expected pattern: none|summary|detailed config.setMetricsLevel(metricsLevel); // Controls the granularity of metrics that are uploaded to CloudWatch. // Greater granularity produces more metrics. // Expected pattern: global|stream|shard config.setMetricsGranularity(metricsGranuality); // The namespace to upload metrics under. config.setMetricsNamespace(metricsNameSpace); return new KinesisProducer(config); }
Example #12
Source File: KinesisPublisher.java From james with Apache License 2.0 | 5 votes |
private KinesisProducer createKinesisProducer(KinesisPublisherConfiguration configuration) { ProducerConfiguration pc = configuration.getProducer(); KinesisProducerConfiguration kc = new KinesisProducerConfiguration(); pc.isAggregationEnabled().ifPresent(kc::setAggregationEnabled); pc.getAggregationMaxCount().ifPresent(kc::setAggregationMaxCount); pc.getAggregationMaxSize().ifPresent(kc::setAggregationMaxSize); pc.getCloudwatchEndpoint().ifPresent(kc::setCloudwatchEndpoint); pc.getCloudwatchPort().ifPresent(kc::setCloudwatchPort); pc.getCollectionMaxCount().ifPresent(kc::setCollectionMaxCount); pc.getCollectionMaxSize().ifPresent(kc::setCollectionMaxSize); pc.getConnectTimeout().ifPresent(kc::setConnectTimeout); pc.isEnableCoreDumps().ifPresent(kc::setEnableCoreDumps); pc.isFailIfThrottled().ifPresent(kc::setFailIfThrottled); pc.getKinesisEndpoint().ifPresent(kc::setKinesisEndpoint); pc.getKinesisPort().ifPresent(kc::setKinesisPort); pc.getLogLevel().ifPresent(kc::setLogLevel); pc.getMaxConnections().ifPresent(kc::setMaxConnections); pc.getMinConnections().ifPresent(kc::setMinConnections); pc.getNativeExecutable().ifPresent(kc::setNativeExecutable); pc.getRateLimit().ifPresent(kc::setRateLimit); pc.getRecordMaxBufferedTime().ifPresent(kc::setRecordMaxBufferedTime); pc.getRecordTtl().ifPresent(kc::setRecordTtl); pc.getRegion().ifPresent(kc::setRegion); pc.getRequestTimeout().ifPresent(kc::setRequestTimeout); pc.getTempDirectory().ifPresent(kc::setTempDirectory); pc.isVerifyCertificate().ifPresent(kc::setVerifyCertificate); try { return new KinesisProducer(kc); } catch (Exception e) { LOG.error("Failed to initialize " + KinesisPublisher.class.getSimpleName(), e); return null; } }
Example #13
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCorrectlySetRegionInProducerConfiguration() { String region = "us-east-1"; Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, region); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals("incorrect region", region, kpc.getRegion()); }
Example #14
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testThreadPoolSizeInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(10, kpc.getThreadPoolSize()); testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(12, kpc.getThreadPoolSize()); }
Example #15
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testRateLimitInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(100, kpc.getRateLimit()); testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(150, kpc.getRateLimit()); }
Example #16
Source File: KinesisConfigUtilTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testThreadingModelInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel()); testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel()); }
Example #17
Source File: KinesisConfigUtil.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Validate configuration properties for {@link FlinkKinesisProducer}, * and return a constructed KinesisProducerConfiguration. */ public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) { checkNotNull(config, "config can not be null"); validateAwsConfiguration(config); if (!config.containsKey(AWSConfigConstants.AWS_REGION)) { // per requirement in Amazon Kinesis Producer Library throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION)); } KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config); kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION)); kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config)); // we explicitly lower the credential refresh delay (default is 5 seconds) // to avoid an ignorable interruption warning that occurs when shutting down the // KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10. kpc.setCredentialsRefreshDelay(100); // Override default values if they aren't specified by users if (!config.containsKey(RATE_LIMIT)) { kpc.setRateLimit(DEFAULT_RATE_LIMIT); } if (!config.containsKey(THREADING_MODEL)) { kpc.setThreadingModel(DEFAULT_THREADING_MODEL); } if (!config.containsKey(THREAD_POOL_SIZE)) { kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE); } return kpc; }
Example #18
Source File: KinesisConfigUtil.java From flink with Apache License 2.0 | 5 votes |
/** * Validate configuration properties for {@link FlinkKinesisProducer}, * and return a constructed KinesisProducerConfiguration. */ public static KinesisProducerConfiguration getValidatedProducerConfiguration(Properties config) { checkNotNull(config, "config can not be null"); validateAwsConfiguration(config); if (!config.containsKey(AWSConfigConstants.AWS_REGION)) { // per requirement in Amazon Kinesis Producer Library throw new IllegalArgumentException(String.format("For FlinkKinesisProducer AWS region ('%s') must be set in the config.", AWSConfigConstants.AWS_REGION)); } KinesisProducerConfiguration kpc = KinesisProducerConfiguration.fromProperties(config); kpc.setRegion(config.getProperty(AWSConfigConstants.AWS_REGION)); kpc.setCredentialsProvider(AWSUtil.getCredentialsProvider(config)); // we explicitly lower the credential refresh delay (default is 5 seconds) // to avoid an ignorable interruption warning that occurs when shutting down the // KPL client. See https://github.com/awslabs/amazon-kinesis-producer/issues/10. kpc.setCredentialsRefreshDelay(100); // Override default values if they aren't specified by users if (!config.containsKey(RATE_LIMIT)) { kpc.setRateLimit(DEFAULT_RATE_LIMIT); } if (!config.containsKey(THREADING_MODEL)) { kpc.setThreadingModel(DEFAULT_THREADING_MODEL); } if (!config.containsKey(THREAD_POOL_SIZE)) { kpc.setThreadPoolSize(DEFAULT_THREAD_POOL_SIZE); } return kpc; }
Example #19
Source File: KinesisConfigUtilTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testRateLimitInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(100, kpc.getRateLimit()); testConfig.setProperty(KinesisConfigUtil.RATE_LIMIT, "150"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(150, kpc.getRateLimit()); }
Example #20
Source File: KinesisConfigUtilTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testThreadingModelInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(KinesisProducerConfiguration.ThreadingModel.POOLED, kpc.getThreadingModel()); testConfig.setProperty(KinesisConfigUtil.THREADING_MODEL, "PER_REQUEST"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(KinesisProducerConfiguration.ThreadingModel.PER_REQUEST, kpc.getThreadingModel()); }
Example #21
Source File: KinesisConfigUtilTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testCorrectlySetRegionInProducerConfiguration() { String region = "us-east-1"; Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, region); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals("incorrect region", region, kpc.getRegion()); }
Example #22
Source File: KinesisConfigUtilTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testThreadPoolSizeInProducerConfiguration() { Properties testConfig = new Properties(); testConfig.setProperty(AWSConfigConstants.AWS_REGION, "us-east-1"); KinesisProducerConfiguration kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(10, kpc.getThreadPoolSize()); testConfig.setProperty(KinesisConfigUtil.THREAD_POOL_SIZE, "12"); kpc = KinesisConfigUtil.getValidatedProducerConfiguration(testConfig); assertEquals(12, kpc.getThreadPoolSize()); }
Example #23
Source File: FlinkKinesisProducer.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Creates a {@link KinesisProducer}. * Exposed so that tests can inject mock producers easily. */ @VisibleForTesting protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) { return new KinesisProducer(producerConfig); }
Example #24
Source File: FlinkKinesisProducerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Override protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) { return mockProducer; }
Example #25
Source File: FlinkKinesisProducerTest.java From flink with Apache License 2.0 | 4 votes |
@Override protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) { return mockProducer; }
Example #26
Source File: FlinkKinesisProducer.java From flink with Apache License 2.0 | 4 votes |
/** * Creates a {@link KinesisProducer}. * Exposed so that tests can inject mock producers easily. */ @VisibleForTesting protected KinesisProducer getKinesisProducer(KinesisProducerConfiguration producerConfig) { return new KinesisProducer(producerConfig); }
Example #27
Source File: FlinkKinesisProducer.java From flink with Apache License 2.0 | 4 votes |
@Override public void open(Configuration parameters) throws Exception { super.open(parameters); schema.open(() -> getRuntimeContext().getMetricGroup().addGroup("user")); // check and pass the configuration properties KinesisProducerConfiguration producerConfig = KinesisConfigUtil.getValidatedProducerConfiguration(configProps); producer = getKinesisProducer(producerConfig); final MetricGroup kinesisMectricGroup = getRuntimeContext().getMetricGroup().addGroup(KINESIS_PRODUCER_METRIC_GROUP); this.backpressureCycles = kinesisMectricGroup.counter(METRIC_BACKPRESSURE_CYCLES); kinesisMectricGroup.gauge(METRIC_OUTSTANDING_RECORDS_COUNT, producer::getOutstandingRecordsCount); backpressureLatch = new TimeoutLatch(); callback = new FutureCallback<UserRecordResult>() { @Override public void onSuccess(UserRecordResult result) { backpressureLatch.trigger(); if (!result.isSuccessful()) { if (failOnError) { // only remember the first thrown exception if (thrownException == null) { thrownException = new RuntimeException("Record was not sent successful"); } } else { LOG.warn("Record was not sent successful"); } } } @Override public void onFailure(Throwable t) { backpressureLatch.trigger(); if (failOnError) { thrownException = t; } else { LOG.warn("An exception occurred while processing a record", t); } } }; if (this.customPartitioner != null) { this.customPartitioner.initialize(getRuntimeContext().getIndexOfThisSubtask(), getRuntimeContext().getNumberOfParallelSubtasks()); } LOG.info("Started Kinesis producer instance for region '{}'", producerConfig.getRegion()); }
Example #28
Source File: KPLClickEventsToKinesis.java From aws-big-data-blog with Apache License 2.0 | 4 votes |
public KPLClickEventsToKinesis(BlockingQueue<ClickEvent> inputQueue) { super(inputQueue); kinesis = new KinesisProducer(new KinesisProducerConfiguration() .setRegion(REGION) .setRecordMaxBufferedTime(5000)); }
Example #29
Source File: AdvancedKPLClickEventsToKinesis.java From aws-big-data-blog with Apache License 2.0 | 4 votes |
protected AdvancedKPLClickEventsToKinesis( BlockingQueue<ClickEvent> inputQueue) { super(inputQueue); kinesis = new KinesisProducer(new KinesisProducerConfiguration() .setRegion(REGION)); }
Example #30
Source File: TalendKinesisProvider.java From components with Apache License 2.0 | 4 votes |
@Override public IKinesisProducer createKinesisProducer(KinesisProducerConfiguration config) { // TODO implement when support Kinesis output, as it only used for write return null; }