com.amazonaws.services.kinesis.clientlibrary.lib.worker.ShutdownReason Java Examples

@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer, ShutdownReason reason) {
    LOG.info("Shutting down record processor with shardId: " + shardId + " with reason " + reason);
    if (isShutdown) {
        LOG.warn("Record processor for shardId: " + shardId + " has been shutdown multiple times.");
        return;
    }
    switch (reason) {
        case TERMINATE:
            emit(checkpointer, transformToOutput(buffer.getRecords()));
            try {
                checkpointer.checkpoint();
            } catch (KinesisClientLibDependencyException | InvalidStateException | ThrottlingException | ShutdownException e) {
                LOG.error(e);
            }
            break;
        case ZOMBIE:
            break;
        default:
            throw new IllegalStateException("invalid shutdown reason");
    }
    emitter.shutdown();
    isShutdown = true;
}
 

@Override
public void shutdown(IRecordProcessorCheckpointer checkpointer, ShutdownReason reason) {
    LOG.info("Shutting down pipelined processor for shard: " + shardId + " with reason:" + reason);
    queueConsumer.shutdown = true;
    try {
        if (queueConsumerExecutor.awaitTermination(maxProcessRecordsWaitTimeMs, TimeUnit.MILLISECONDS)) {
            List<Record> records = new ArrayList<Record>();
            recordQueue.drainTo(records);
            // No need to protect the checkpointer any longer. Record processing is in sync with record fetching.
            recordProcessor.processRecords(records, checkpointer);
            recordProcessor.shutdown(checkpointer, reason);
        } else {
            LOG.warn("Queue consumer took longer than " + maxProcessRecordsWaitTimeMs + " ms to complete. Shutdown task failed.");
        }
    } catch (InterruptedException e) {
        LOG.error("Interrupted while draining queue", e);
        Thread.currentThread().interrupt();
    }
}
 

/**
 * expect nothing to happen on ShutdownReason.ZOMBIE
 */
@Test
public void testShutdownZombie() {
    // reset the control to mock new behavior
    control.reset();
    // expect shutdown to be called on emitter
    emitter.shutdown();
    EasyMock.expectLastCall();

    // Prepare controller for method call
    control.replay();

    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.shutdown(checkpointer, ShutdownReason.ZOMBIE);

    control.verify();
}
 

@Override
public void shutdown(ShutdownInput shutdownInput) {
    if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) {
        try {
            shutdownInput.getCheckpointer().checkpoint();
        }
        catch (Exception e) {
            e.printStackTrace();
        }
    }

}
 

/**
 * {@inheritDoc}
 */
@Override
public void shutdown(final IRecordProcessorCheckpointer checkpointer, final ShutdownReason reason) {
    LOG.info("Shutting down record processor for shard: " + kinesisShardId);
    // Important to checkpoint after reaching end of shard, so we can start processing data from child shards.
    if (reason == ShutdownReason.TERMINATE) {
        checkpoint(checkpointer);
    }
}
 

@Override
public void shutdown(ShutdownInput shutdownInput) {
    if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) {
        try {
            shutdownInput.getCheckpointer().checkpoint();
        }
        catch (Exception e) {
            e.printStackTrace();
        }
    }
    
}
 

/**
 * Invoked by the Amazon Kinesis Client Library to indicate it will no longer send data records to this
 * RecordProcessor instance.
 *
 * @param shutdownInput Provides information and capabilities (eg checkpointing) related to shutdown of this record
 *        processor.
 */
@Override
public void shutdown(ShutdownInput shutdownInput) {
  LOG.info("Shutting down {} with reason:{}", this, shutdownInput.getShutdownReason());

  Validate.isTrue(!shutdownRequested, String.format("KCL called shutdown more than once for processor %s.", this));
  shutdownRequested = true;
  // shutdown reason TERMINATE indicates that the shard is closed due to re-sharding. It also indicates that all the
  // records from the shard have been delivered to the consumer and the consumer is expected to checkpoint the
  // progress.
  if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) {
    // We need to ensure that all records are processed and checkpointed before going ahead and marking the processing
    // complete by calling checkpoint() on KCL. We need to checkpoint the completion state for parent shard, for KCL
    // to consume from the child shard(s).
    try {
      LOG.info("Waiting for all the records for {} to be processed.", this);
      // Let's poll periodically and block until the last processed record is checkpointed. Also handle the case
      // where there are no records received for the processor, in which case the lastProcessedRecordSeqNumber will
      // be null.
      while (lastProcessedRecordSeqNumber != null
          && !lastProcessedRecordSeqNumber.equals(lastCheckpointedRecordSeqNumber)) {
        Thread.sleep(POLL_INTERVAL_DURING_PARENT_SHARD_SHUTDOWN_MS);
      }
      LOG.info("Final checkpoint for {} before shutting down.", this);
      shutdownInput.getCheckpointer().checkpoint();
    } catch (Exception e) {
      LOG.warn("An error occurred while committing the final checkpoint in the parent shard {}", this, e);
    }
  }
  listener.onShutdown(ssp);
}
 

static void shutDownProcessor(KinesisRecordProcessor processor, ShutdownReason reason) {
  try {
    ShutdownInput shutdownInput = Mockito.mock(ShutdownInput.class);
    when(shutdownInput.getShutdownReason()).thenReturn(reason);
    when(shutdownInput.getCheckpointer()).thenReturn(getCheckpointer(processor));
    processor.shutdown(shutdownInput);
  } catch (Exception ex) {
    throw new RuntimeException(ex);
  }
}
 

/**
 * We don't checkpoint on SHUTDOWN_REQUESTED because we currently always
 * checkpoint each batch in {@link #processRecords}.
 *
 * @param shutdownInput {@inheritDoc}
 */
@Override
public void shutdown(ShutdownInput shutdownInput) {
  LOG.info("Shutting down record processor for shard: {}", shardId);

  if (ShutdownReason.TERMINATE.equals(shutdownInput.getShutdownReason())) {
    // Shard is closed / finished processing. Checkpoint all processing up to here.
    try {
      shutdownInput.getCheckpointer().checkpoint();
      LOG.debug("Checkpointed due to record processor shutdown request.");
    } catch (InvalidStateException | ShutdownException e) {
      LOG.error("Error checkpointing batch: {}", e.toString(), e);
    }
  }
}
 

/**
 * expect buffer flush, emit and checkpoint to happen on ShutdownReason.TERMINATE
 */
@Test
public void testShutdownTerminate() throws IOException, KinesisClientLibDependencyException, InvalidStateException, ThrottlingException, ShutdownException {
    // reset the control to mock new behavior
    control.reset();

    // expect flush cycle.
    // Get records from buffer, emit, clear, then checkpoint
    EasyMock.expect(buffer.getRecords()).andReturn(Collections.emptyList());
    EasyMock.expect(emitter.emit(EasyMock.anyObject(UnmodifiableBuffer.class))).andReturn(
            Collections.emptyList());
    buffer.getLastSequenceNumber();
    EasyMock.expectLastCall().andReturn(null);
    buffer.clear();
    EasyMock.expectLastCall();
    checkpointer.checkpoint();
    EasyMock.expectLastCall();

    // expect shutdown to be called on emitter
    emitter.shutdown();
    EasyMock.expectLastCall();

    // Prepare controller for method call
    control.replay();

    KinesisConnectorRecordProcessor<Object, Object> kcrp = new KinesisConnectorRecordProcessor<Object, Object>(
            buffer, filter, emitter, transformer, configuration);
    kcrp.shutdown(checkpointer, ShutdownReason.TERMINATE);

    control.verify();
}
 

protected IRecordProcessor createStreamProcessor() {
	return new IRecordProcessor() {

		@Override
		public void initialize(InitializationInput initializationInput) {
		}

		public List<Record> extractDynamoStreamRecords(List<com.amazonaws.services.kinesis.model.Record> kinesisRecords) {
			List<Record> dynamoRecords = new ArrayList<>(kinesisRecords.size());

			for(com.amazonaws.services.kinesis.model.Record kinesisRecord : kinesisRecords) {
				if (kinesisRecord instanceof RecordAdapter) {
					Record dynamoRecord = ((RecordAdapter) kinesisRecord).getInternalObject();
					dynamoRecords.add(dynamoRecord);
				}
			}

			return dynamoRecords;
		}

		@Override
		public void processRecords(ProcessRecordsInput processRecordsInput) {
			List<Record> records = extractDynamoStreamRecords(processRecordsInput.getRecords());

			DynamoDBTableReplicator.this.processRecords(records);

			checkpoint(processRecordsInput.getCheckpointer());
		}

		@Override
		public void shutdown(ShutdownInput shutdownInput) {
			if (shutdownInput.getShutdownReason() == ShutdownReason.TERMINATE) {
				checkpoint(shutdownInput.getCheckpointer());
			}
		}

		void checkpoint(IRecordProcessorCheckpointer checkpointer) {
			try {
				checkpointer.checkpoint();
			} catch (KinesisClientLibDependencyException|InvalidStateException|ThrottlingException|ShutdownException e) {
				LOG.warn(e);
			}
		}
	};
}
 

/**
 * Helper to simulate and test the life-cycle of record processing from a kinesis stream with a given number of shards
 * 1. Creation of record processors.
 * 2. Initialization of record processors.
 * 3. Processing records via record processors.
 * 4. Calling checkpoint on record processors.
 * 5. Shutting down (due to re-assignment or lease expiration) record processors.
 */
private void testProcessRecordsHelper(String system, String stream, int numShards, int numRecordsPerShard)
    throws InterruptedException, NoSuchFieldException, IllegalAccessException {

  KinesisConfig kConfig = new KinesisConfig(new MapConfig());
  // Create consumer
  KinesisSystemConsumer consumer = new KinesisSystemConsumer(system, kConfig, new NoOpMetricsRegistry());
  initializeMetrics(consumer, stream);

  List<SystemStreamPartition> ssps = new LinkedList<>();
  IntStream.range(0, numShards)
      .forEach(p -> {
        SystemStreamPartition ssp = new SystemStreamPartition(system, stream, new Partition(p));
        ssps.add(ssp);
      });
  ssps.forEach(ssp -> consumer.register(ssp, SYSTEM_CONSUMER_REGISTER_OFFSET));

  // Create Kinesis record processor factory
  IRecordProcessorFactory factory = consumer.createRecordProcessorFactory(stream);

  // Create and initialize Kinesis record processor
  Map<String, KinesisRecordProcessor> processorMap = createAndInitProcessors(factory, numShards);
  List<KinesisRecordProcessor> processorList = new ArrayList<>(processorMap.values());

  // Generate records to Kinesis record processor
  Map<KinesisRecordProcessor, List<Record>> inputRecordMap = generateRecords(numRecordsPerShard, processorList);

  // Verification steps

  // Read events from the BEM queue
  Map<SystemStreamPartition, List<IncomingMessageEnvelope>> messages =
      readEvents(new HashSet<>(ssps), consumer, numRecordsPerShard);
  if (numRecordsPerShard > 0) {
    Assert.assertEquals(messages.size(), numShards);
  } else {
    // No input records and hence no messages
    Assert.assertEquals(messages.size(), 0);
    return;
  }

  Map<SystemStreamPartition, KinesisRecordProcessor> sspToProcessorMap = getProcessorMap(consumer);
  ssps.forEach(ssp -> {
    try {
      KinesisRecordProcessor processor = sspToProcessorMap.get(ssp);

      // Verify that the read messages are received in order and are the same as input records
      Assert.assertEquals(messages.get(ssp).size(), numRecordsPerShard);
      List<IncomingMessageEnvelope> envelopes = messages.get(ssp);
      List<Record> inputRecords = inputRecordMap.get(processor);
      verifyRecords(envelopes, inputRecords, processor.getShardId());

      // Call checkpoint on consumer and verify that the checkpoint is called with the right offset
      IncomingMessageEnvelope lastEnvelope = envelopes.get(envelopes.size() - 1);
      consumer.afterCheckpoint(Collections.singletonMap(ssp, lastEnvelope.getOffset()));
      ArgumentCaptor<String> argument = ArgumentCaptor.forClass(String.class);
      verify(getCheckpointer(processor)).checkpoint(argument.capture());
      Assert.assertEquals(inputRecords.get(inputRecords.size() - 1).getSequenceNumber(), argument.getValue());

      // Call shutdown (with ZOMBIE reason) on processor and verify if shutdown freed the ssp mapping
      shutDownProcessor(processor, ShutdownReason.ZOMBIE);
      Assert.assertFalse(sspToProcessorMap.containsValue(processor));
      Assert.assertTrue(isSspAvailable(consumer, ssp));
    } catch (NoSuchFieldException | IllegalAccessException | InvalidStateException | ShutdownException ex) {
      throw new RuntimeException(ex);
    }
  });
}