org.apache.flink.streaming.connectors.kinesis.metrics.ShardMetricsReporter Java Examples
The following examples show how to use
org.apache.flink.streaming.connectors.kinesis.metrics.ShardMetricsReporter.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DynamoDBStreamsDataFetcher.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Create a new DynamoDB streams shard consumer. * * @param subscribedShardStateIndex the state index of the shard this consumer is subscribed to * @param handle stream handle * @param lastSeqNum last sequence number * @param shardMetricsReporter the reporter to report metrics to * @return */ @Override protected ShardConsumer createShardConsumer( Integer subscribedShardStateIndex, StreamShardHandle handle, SequenceNumber lastSeqNum, ShardMetricsReporter shardMetricsReporter) { return new ShardConsumer( this, subscribedShardStateIndex, handle, lastSeqNum, DynamoDBStreamsProxy.create(getConsumerConfiguration()), shardMetricsReporter); }
Example #2
Source File: KinesisDataFetcher.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Registers a metric group associated with the shard id of the provided {@link KinesisStreamShardState shardState}. * * @return a {@link ShardMetricsReporter} that can be used to update metric values */ private static ShardMetricsReporter registerShardMetrics(MetricGroup metricGroup, KinesisStreamShardState shardState) { ShardMetricsReporter shardMetrics = new ShardMetricsReporter(); MetricGroup streamShardMetricGroup = metricGroup .addGroup( KinesisConsumerMetricConstants.STREAM_METRICS_GROUP, shardState.getStreamShardHandle().getStreamName()) .addGroup( KinesisConsumerMetricConstants.SHARD_METRICS_GROUP, shardState.getStreamShardHandle().getShard().getShardId()); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MILLIS_BEHIND_LATEST_GAUGE, shardMetrics::getMillisBehindLatest); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MAX_RECORDS_PER_FETCH, shardMetrics::getMaxNumberOfRecordsPerFetch); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_AGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfAggregatedRecords); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_DEAGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfDeaggregatedRecords); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.AVG_RECORD_SIZE_BYTES, shardMetrics::getAverageRecordSizeBytes); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.BYTES_PER_READ, shardMetrics::getBytesPerRead); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.RUNTIME_LOOP_NANOS, shardMetrics::getRunLoopTimeNanos); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.LOOP_FREQUENCY_HZ, shardMetrics::getLoopFrequencyHz); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.SLEEP_TIME_MILLIS, shardMetrics::getSleepTimeMillis); return shardMetrics; }
Example #3
Source File: KinesisDataFetcher.java From flink with Apache License 2.0 | 6 votes |
/** * Registers a metric group associated with the shard id of the provided {@link KinesisStreamShardState shardState}. * * @return a {@link ShardMetricsReporter} that can be used to update metric values */ private static ShardMetricsReporter registerShardMetrics(MetricGroup metricGroup, KinesisStreamShardState shardState) { ShardMetricsReporter shardMetrics = new ShardMetricsReporter(); MetricGroup streamShardMetricGroup = metricGroup .addGroup( KinesisConsumerMetricConstants.STREAM_METRICS_GROUP, shardState.getStreamShardHandle().getStreamName()) .addGroup( KinesisConsumerMetricConstants.SHARD_METRICS_GROUP, shardState.getStreamShardHandle().getShard().getShardId()); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MILLIS_BEHIND_LATEST_GAUGE, shardMetrics::getMillisBehindLatest); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MAX_RECORDS_PER_FETCH, shardMetrics::getMaxNumberOfRecordsPerFetch); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_AGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfAggregatedRecords); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_DEAGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfDeaggregatedRecords); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.AVG_RECORD_SIZE_BYTES, shardMetrics::getAverageRecordSizeBytes); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.BYTES_PER_READ, shardMetrics::getBytesPerRead); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.RUNTIME_LOOP_NANOS, shardMetrics::getRunLoopTimeNanos); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.LOOP_FREQUENCY_HZ, shardMetrics::getLoopFrequencyHz); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.SLEEP_TIME_MILLIS, shardMetrics::getSleepTimeMillis); return shardMetrics; }
Example #4
Source File: KinesisDataFetcher.java From flink with Apache License 2.0 | 6 votes |
/** * Create a new shard consumer. * Override this method to customize shard consumer behavior in subclasses. * @param subscribedShardStateIndex the state index of the shard this consumer is subscribed to * @param subscribedShard the shard this consumer is subscribed to * @param lastSequenceNum the sequence number in the shard to start consuming * @param shardMetricsReporter the reporter to report metrics to * @return shard consumer */ protected ShardConsumer<T> createShardConsumer( Integer subscribedShardStateIndex, StreamShardHandle subscribedShard, SequenceNumber lastSequenceNum, ShardMetricsReporter shardMetricsReporter, KinesisDeserializationSchema<T> shardDeserializer) { return new ShardConsumer<>( this, subscribedShardStateIndex, subscribedShard, lastSequenceNum, this.kinesisProxyFactory.create(configProps), shardMetricsReporter, shardDeserializer); }
Example #5
Source File: DynamoDBStreamsDataFetcher.java From flink with Apache License 2.0 | 6 votes |
/** * Create a new DynamoDB streams shard consumer. * * @param subscribedShardStateIndex the state index of the shard this consumer is subscribed to * @param handle stream handle * @param lastSeqNum last sequence number * @param shardMetricsReporter the reporter to report metrics to * @return */ @Override protected ShardConsumer<T> createShardConsumer( Integer subscribedShardStateIndex, StreamShardHandle handle, SequenceNumber lastSeqNum, ShardMetricsReporter shardMetricsReporter, KinesisDeserializationSchema<T> shardDeserializer) { return new ShardConsumer( this, subscribedShardStateIndex, handle, lastSeqNum, DynamoDBStreamsProxy.create(getConsumerConfiguration()), shardMetricsReporter, shardDeserializer); }
Example #6
Source File: DynamoDBStreamsDataFetcher.java From flink with Apache License 2.0 | 6 votes |
/** * Create a new DynamoDB streams shard consumer. * * @param subscribedShardStateIndex the state index of the shard this consumer is subscribed to * @param handle stream handle * @param lastSeqNum last sequence number * @param shardMetricsReporter the reporter to report metrics to * @return */ @Override protected ShardConsumer createShardConsumer( Integer subscribedShardStateIndex, StreamShardHandle handle, SequenceNumber lastSeqNum, ShardMetricsReporter shardMetricsReporter) { return new ShardConsumer( this, subscribedShardStateIndex, handle, lastSeqNum, DynamoDBStreamsProxy.create(getConsumerConfiguration()), shardMetricsReporter); }
Example #7
Source File: KinesisDataFetcher.java From flink with Apache License 2.0 | 6 votes |
/** * Registers a metric group associated with the shard id of the provided {@link KinesisStreamShardState shardState}. * * @return a {@link ShardMetricsReporter} that can be used to update metric values */ private static ShardMetricsReporter registerShardMetrics(MetricGroup metricGroup, KinesisStreamShardState shardState) { ShardMetricsReporter shardMetrics = new ShardMetricsReporter(); MetricGroup streamShardMetricGroup = metricGroup .addGroup( KinesisConsumerMetricConstants.STREAM_METRICS_GROUP, shardState.getStreamShardHandle().getStreamName()) .addGroup( KinesisConsumerMetricConstants.SHARD_METRICS_GROUP, shardState.getStreamShardHandle().getShard().getShardId()); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MILLIS_BEHIND_LATEST_GAUGE, shardMetrics::getMillisBehindLatest); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.MAX_RECORDS_PER_FETCH, shardMetrics::getMaxNumberOfRecordsPerFetch); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_AGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfAggregatedRecords); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.NUM_DEAGGREGATED_RECORDS_PER_FETCH, shardMetrics::getNumberOfDeaggregatedRecords); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.AVG_RECORD_SIZE_BYTES, shardMetrics::getAverageRecordSizeBytes); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.BYTES_PER_READ, shardMetrics::getBytesPerRead); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.RUNTIME_LOOP_NANOS, shardMetrics::getRunLoopTimeNanos); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.LOOP_FREQUENCY_HZ, shardMetrics::getLoopFrequencyHz); streamShardMetricGroup.gauge(KinesisConsumerMetricConstants.SLEEP_TIME_MILLIS, shardMetrics::getSleepTimeMillis); return shardMetrics; }
Example #8
Source File: KinesisDataFetcher.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Create a new shard consumer. * Override this method to customize shard consumer behavior in subclasses. * @param subscribedShardStateIndex the state index of the shard this consumer is subscribed to * @param subscribedShard the shard this consumer is subscribed to * @param lastSequenceNum the sequence number in the shard to start consuming * @param shardMetricsReporter the reporter to report metrics to * @return shard consumer */ protected ShardConsumer createShardConsumer( Integer subscribedShardStateIndex, StreamShardHandle subscribedShard, SequenceNumber lastSequenceNum, ShardMetricsReporter shardMetricsReporter) { return new ShardConsumer<>( this, subscribedShardStateIndex, subscribedShard, lastSequenceNum, this.kinesisProxyFactory.create(configProps), shardMetricsReporter); }
Example #9
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedState() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L), new ShardMetricsReporter()).run(); assertEquals(1000, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #10
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testMetricsReporting() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState( KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter(); long millisBehindLatest = 500L; new ShardConsumer<>( fetcher, 0, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest), shardMetricsReporter).run(); // the millisBehindLatest metric should have been reported assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest()); }
Example #11
Source File: KinesisDataFetcher.java From flink with Apache License 2.0 | 5 votes |
/** * Create a new shard consumer. * Override this method to customize shard consumer behavior in subclasses. * @param subscribedShardStateIndex the state index of the shard this consumer is subscribed to * @param subscribedShard the shard this consumer is subscribed to * @param lastSequenceNum the sequence number in the shard to start consuming * @param shardMetricsReporter the reporter to report metrics to * @return shard consumer */ protected ShardConsumer createShardConsumer( Integer subscribedShardStateIndex, StreamShardHandle subscribedShard, SequenceNumber lastSequenceNum, ShardMetricsReporter shardMetricsReporter) { return new ShardConsumer<>( this, subscribedShardStateIndex, subscribedShard, lastSequenceNum, this.kinesisProxyFactory.create(configProps), shardMetricsReporter); }
Example #12
Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedState() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L), new ShardMetricsReporter()).run(); assertEquals(1000, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #13
Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testMetricsReporting() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState( KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter(); long millisBehindLatest = 500L; new ShardConsumer<>( fetcher, 0, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest), shardMetricsReporter).run(); // the millisBehindLatest metric should have been reported assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest()); }
Example #14
Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() { Properties consumerProperties = new Properties(); consumerProperties.put("flink.shard.adaptivereads", "true"); StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, consumerProperties, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), // Initial number of records to fetch --> 10 FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L), new ShardMetricsReporter()).run(); // Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb // Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40 // Total number of records = 10 + 40 = 50 assertEquals(50, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #15
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), // Get a total of 1000 records with 9 getRecords() calls, // and the 7th getRecords() call will encounter an unexpected expired shard iterator FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator( 1000, 9, 7, 500L), new ShardMetricsReporter()).run(); assertEquals(1000, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #16
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() { Properties consumerProperties = new Properties(); consumerProperties.put("flink.shard.adaptivereads", "true"); StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, consumerProperties, new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), // Initial number of records to fetch --> 10 FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L), new ShardMetricsReporter()).run(); // Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb // Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40 // Total number of records = 10 + 40 = 50 assertEquals(50, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #17
Source File: ShardConsumerTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), new KinesisDeserializationSchemaWrapper<>(new SimpleStringSchema()), 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), // Get a total of 1000 records with 9 getRecords() calls, // and the 7th getRecords() call will encounter an unexpected expired shard iterator FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator( 1000, 9, 7, 500L), new ShardMetricsReporter()).run(); assertEquals(1000, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #18
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testMetricsReporting() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState( KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>( new SimpleStringSchema()); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), deserializationSchema, 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); ShardMetricsReporter shardMetricsReporter = new ShardMetricsReporter(); long millisBehindLatest = 500L; new ShardConsumer<>( fetcher, 0, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, millisBehindLatest), shardMetricsReporter, deserializationSchema) .run(); // the millisBehindLatest metric should have been reported assertEquals(millisBehindLatest, shardMetricsReporter.getMillisBehindLatest()); }
Example #19
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedState() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>( new SimpleStringSchema()); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), deserializationSchema, 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCalls(1000, 9, 500L), new ShardMetricsReporter(), deserializationSchema) .run(); assertEquals(1000, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #20
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithUnexpectedExpiredIterator() { StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>( new SimpleStringSchema()); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, new Properties(), deserializationSchema, 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), // Get a total of 1000 records with 9 getRecords() calls, // and the 7th getRecords() call will encounter an unexpected expired shard iterator FakeKinesisBehavioursFactory.totalNumOfRecordsAfterNumOfGetRecordsCallsWithUnexpectedExpiredIterator( 1000, 9, 7, 500L), new ShardMetricsReporter(), deserializationSchema) .run(); assertEquals(1000, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }
Example #21
Source File: ShardConsumerTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testCorrectNumOfCollectedRecordsAndUpdatedStateWithAdaptiveReads() { Properties consumerProperties = new Properties(); consumerProperties.put("flink.shard.adaptivereads", "true"); StreamShardHandle fakeToBeConsumedShard = getMockStreamShard("fakeStream", 0); LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest = new LinkedList<>(); subscribedShardsStateUnderTest.add( new KinesisStreamShardState(KinesisDataFetcher.convertToStreamShardMetadata(fakeToBeConsumedShard), fakeToBeConsumedShard, new SequenceNumber("fakeStartingState"))); TestSourceContext<String> sourceContext = new TestSourceContext<>(); KinesisDeserializationSchemaWrapper<String> deserializationSchema = new KinesisDeserializationSchemaWrapper<>( new SimpleStringSchema()); TestableKinesisDataFetcher<String> fetcher = new TestableKinesisDataFetcher<>( Collections.singletonList("fakeStream"), sourceContext, consumerProperties, deserializationSchema, 10, 2, new AtomicReference<>(), subscribedShardsStateUnderTest, KinesisDataFetcher.createInitialSubscribedStreamsToLastDiscoveredShardsState(Collections.singletonList("fakeStream")), Mockito.mock(KinesisProxyInterface.class)); int shardIndex = fetcher.registerNewSubscribedShardState(subscribedShardsStateUnderTest.get(0)); new ShardConsumer<>( fetcher, shardIndex, subscribedShardsStateUnderTest.get(0).getStreamShardHandle(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum(), // Initial number of records to fetch --> 10 FakeKinesisBehavioursFactory.initialNumOfRecordsAfterNumOfGetRecordsCallsWithAdaptiveReads(10, 2, 500L), new ShardMetricsReporter(), deserializationSchema) .run(); // Avg record size for first batch --> 10 * 10 Kb/10 = 10 Kb // Number of records fetched in second batch --> 2 Mb/10Kb * 5 = 40 // Total number of records = 10 + 40 = 50 assertEquals(50, sourceContext.getCollectedOutputs().size()); assertEquals( SentinelSequenceNumber.SENTINEL_SHARD_ENDING_SEQUENCE_NUM.get(), subscribedShardsStateUnderTest.get(0).getLastProcessedSequenceNum()); }