org.apache.kafka.common.metrics.Metrics Java Examples

/**
 * Creates a new network client with the given properties.
 *
 * @return A new network client with the given properties.
 */
NetworkClient createNetworkClient(long connectionMaxIdleMS,
                                  Metrics metrics,
                                  Time time,
                                  String metricGrpPrefix,
                                  ChannelBuilder channelBuilder,
                                  Metadata metadata,
                                  String clientId,
                                  int maxInFlightRequestsPerConnection,
                                  long reconnectBackoffMs,
                                  long reconnectBackoffMax,
                                  int socketSendBuffer,
                                  int socketReceiveBuffer,
                                  int defaultRequestTimeoutMs,
                                  boolean discoverBrokerVersions,
                                  ApiVersions apiVersions);
 

public static void initialize() {
  MetricConfig metricConfig = new MetricConfig()
      .samples(100)
      .timeWindow(
          1000,
          TimeUnit.MILLISECONDS
      );
  List<MetricsReporter> reporters = new ArrayList<>();
  reporters.add(new JmxReporter("io.confluent.ksql.metrics"));
  // Replace all static contents other than Time to ensure they are cleaned for tests that are
  // not aware of the need to initialize/cleanup this test, in case test processes are reused.
  // Tests aware of the class clean everything up properly to get the state into a clean state,
  // a full, fresh instantiation here ensures something like KsqlEngineMetricsTest running after
  // another test that used MetricsCollector without running cleanUp will behave correctly.
  metrics = new Metrics(metricConfig, reporters, new SystemTime());
  collectorMap = new ConcurrentHashMap<>();
}
 

public ClusterTopicManipulationService(String name, AdminClient adminClient) {
  LOGGER.info("ClusterTopicManipulationService constructor initiated {}", this.getClass().getName());

  _isOngoingTopicCreationDone = true;
  _isOngoingTopicDeletionDone = true;
  _adminClient = adminClient;
  _executor = Executors.newSingleThreadScheduledExecutor();
  _reportIntervalSecond = Duration.ofSeconds(1);
  _running = new AtomicBoolean(false);
  _configDefinedServiceName = name;
  // TODO: instantiate a new instance of ClusterTopicManipulationMetrics(..) here.

  MetricConfig metricConfig = new MetricConfig().samples(60).timeWindow(1000, TimeUnit.MILLISECONDS);
  List<MetricsReporter> reporters = new ArrayList<>();
  reporters.add(new JmxReporter(Service.JMX_PREFIX));
  Metrics metrics = new Metrics(metricConfig, reporters, new SystemTime());

  Map<String, String> tags = new HashMap<>();
  tags.put("name", name);
  _clusterTopicManipulationMetrics = new ClusterTopicManipulationMetrics(metrics, tags);
}
 

/**
 *
 * @param metrics a named, numerical measurement. sensor is a handle to record numerical measurements as they occur.
 * @param tags metrics/sensor's tags
 */
public ClusterTopicManipulationMetrics(final Metrics metrics, final Map<String, String> tags) {
  super(metrics, tags);
  _topicCreationSensor = metrics.sensor("topic-creation-metadata-propagation");
  _topicDeletionSensor = metrics.sensor("topic-deletion-metadata-propagation");
  _topicCreationSensor.add(new MetricName("topic-creation-metadata-propagation-ms-avg", METRIC_GROUP_NAME,
      "The average propagation duration in ms of propagating topic creation data and metadata to all brokers in the cluster",
      tags), new Avg());
  _topicCreationSensor.add(new MetricName("topic-creation-metadata-propagation-ms-max", METRIC_GROUP_NAME,
      "The maximum propagation time in ms of propagating topic creation data and metadata to all brokers in the cluster",
      tags), new Max());
  _topicDeletionSensor.add(new MetricName("topic-deletion-metadata-propagation-ms-avg", METRIC_GROUP_NAME,
      "The average propagation duration in milliseconds of propagating the topic deletion data and metadata "
          + "across all the brokers in the cluster.", tags), new Avg());
  _topicDeletionSensor.add(new MetricName("topic-deletion-metadata-propagation-ms-max", METRIC_GROUP_NAME,
      "The maximum propagation time in milliseconds of propagating the topic deletion data and metadata "
          + "across all the brokers in the cluster.", tags), new Max());

  LOGGER.debug("{} constructor was initialized successfully.", "ClusterTopicManipulationMetrics");
}
 

/**
 * Metrics for Calculating the offset commit availability of a consumer.
 * @param metrics the commit offset metrics
 * @param tags the tags associated, i.e) kmf.services:name=single-cluster-monitor
 */
public CommitAvailabilityMetrics(final Metrics metrics, final Map<String, String> tags) {
  LOG.info("{} called.", this.getClass().getSimpleName());
  _offsetsCommitted = metrics.sensor("offsets-committed");
  _offsetsCommitted.add(new MetricName("offsets-committed-total", METRIC_GROUP_NAME,
      "The total number of offsets per second that are committed.", tags), new Total());

  _failedCommitOffsets = metrics.sensor("failed-commit-offsets");
  _failedCommitOffsets.add(new MetricName("failed-commit-offsets-avg", METRIC_GROUP_NAME,
      "The average number of offsets per second that have failed.", tags), new Rate());
  _failedCommitOffsets.add(new MetricName("failed-commit-offsets-total", METRIC_GROUP_NAME,
      "The total number of offsets per second that have failed.", tags), new Total());

  metrics.addMetric(new MetricName("offsets-committed-avg", METRIC_GROUP_NAME, "The average offset commits availability.", tags),
    (MetricConfig config, long now) -> {
      Object offsetCommitTotal = metrics.metrics().get(metrics.metricName("offsets-committed-total", METRIC_GROUP_NAME, tags)).metricValue();
      Object offsetCommitFailTotal = metrics.metrics().get(metrics.metricName("failed-commit-offsets-total", METRIC_GROUP_NAME, tags)).metricValue();
      if (offsetCommitTotal != null && offsetCommitFailTotal != null) {
        double offsetsCommittedCount = (double) offsetCommitTotal;
        double offsetsCommittedErrorCount = (double) offsetCommitFailTotal;
        return offsetsCommittedCount / (offsetsCommittedCount + offsetsCommittedErrorCount);
      } else {
        return 0;
      }
    });
}
 

@Test
public void shouldDisplayRateThroughput() throws Exception {

  ConsumerCollector collector = new ConsumerCollector();//
  collector.configure(new Metrics(), "group", new SystemTime());

  for (int i = 0; i < 100; i++){

    Map<TopicPartition, List<ConsumerRecord<Object, Object>>> records = ImmutableMap.of(
            new TopicPartition(TEST_TOPIC, 1), Arrays.asList(
                    new ConsumerRecord<>(TEST_TOPIC, 1, i,  1l, TimestampType.CREATE_TIME,  1l, 10, 10, "key", "1234567890")) );
    ConsumerRecords<Object, Object> consumerRecords = new ConsumerRecords<>(records);

    collector.onConsume(consumerRecords);
  }

  Collection<TopicSensors.Stat> stats = collector.stats(TEST_TOPIC, false);
  assertNotNull(stats);

  assertThat( stats.toString(), containsString("name=consumer-messages-per-sec,"));
  assertThat( stats.toString(), containsString("total-messages, value=100.0"));
}
 

@Test
public void commitAvailabilityTest() throws Exception {
  ConsumeService consumeService = consumeService();
  Metrics metrics = consumeServiceMetrics(consumeService);

  Assert.assertNotNull(metrics.metrics().get(metrics.metricName("offsets-committed-total", METRIC_GROUP_NAME, tags)).metricValue());
  Assert.assertEquals(metrics.metrics().get(metrics.metricName("offsets-committed-total", METRIC_GROUP_NAME, tags)).metricValue(), 0.0);

  /* Should start */
  consumeService.startConsumeThreadForTesting();
  Assert.assertTrue(consumeService.isRunning());

  /* in milliseconds */
  long threadStartDelay = TimeUnit.SECONDS.toMillis(THREAD_START_DELAY_SECONDS);

  /* Thread.sleep safe to do here instead of ScheduledExecutorService
  *  We want to sleep current thread so that consumeService can start running for enough seconds. */
  Thread.sleep(threadStartDelay);
  Assert.assertNotNull(metrics.metrics().get(metrics.metricName("offsets-committed-total", METRIC_GROUP_NAME, tags)).metricValue());
  Assert.assertNotNull(metrics.metrics().get(metrics.metricName("failed-commit-offsets-total", METRIC_GROUP_NAME,
      tags)).metricValue());
  Assert.assertEquals(metrics.metrics().get(metrics.metricName("failed-commit-offsets-total", METRIC_GROUP_NAME, tags)).metricValue(), 0.0);
  Assert.assertNotEquals(metrics.metrics().get(metrics.metricName("offsets-committed-total", METRIC_GROUP_NAME, tags)).metricValue(), 0.0);
  shutdownConsumeService(consumeService);
}
 

@Test
public void commitLatencyTest() throws Exception {
  CommitLatencyMetrics commitLatencyMetrics = Mockito.mock(CommitLatencyMetrics.class);
  Assert.assertNotNull(commitLatencyMetrics);

  ConsumeService consumeService = consumeService();
  Metrics metrics = consumeServiceMetrics(consumeService);

  Assert.assertNull(metrics.metrics().get(metrics.metricName("commit-offset-latency-ms-avg", METRIC_GROUP_NAME, tags)));
  Assert.assertNull(metrics.metrics().get(metrics.metricName("commit-offset-latency-ms-max", METRIC_GROUP_NAME, tags)));

  /* Should start */
  consumeService.startConsumeThreadForTesting();
  Assert.assertTrue(consumeService.isRunning());

  /* in milliseconds */
  long threadStartDelay = TimeUnit.SECONDS.toMillis(THREAD_START_DELAY_SECONDS);

  /* Thread.sleep safe to do here instead of ScheduledExecutorService
   *  We want to sleep current thread so that consumeService can start running for enough seconds. */
  Thread.sleep(threadStartDelay);

  shutdownConsumeService(consumeService);
}
 

/**
 * Metrics for Calculating the offset commit latency of a consumer.
 * @param metrics the commit offset metrics
 * @param tags the tags associated, i.e) kmf.services:name=single-cluster-monitor
 */
public CommitLatencyMetrics(Metrics metrics, Map<String, String> tags, int latencyPercentileMaxMs,
    int latencyPercentileGranularityMs) {
  _inProgressCommit = false;
  _commitOffsetLatency = metrics.sensor("commit-offset-latency");
  _commitOffsetLatency.add(new MetricName("commit-offset-latency-ms-avg", METRIC_GROUP_NAME, "The average latency in ms of committing offset", tags), new Avg());
  _commitOffsetLatency.add(new MetricName("commit-offset-latency-ms-max", METRIC_GROUP_NAME, "The maximum latency in ms of committing offset", tags), new Max());

  if (latencyPercentileGranularityMs == 0) {
    throw new IllegalArgumentException("The latency percentile granularity was incorrectly passed a zero value.");
  }

  // 2 extra buckets exist which are respectively designated for values which are less than 0.0 or larger than max.
  int bucketNum = latencyPercentileMaxMs / latencyPercentileGranularityMs + 2;
  int sizeInBytes = bucketNum * 4;
  _commitOffsetLatency.add(new Percentiles(sizeInBytes, latencyPercentileMaxMs, Percentiles.BucketSizing.CONSTANT,
      new Percentile(new MetricName("commit-offset-latency-ms-99th", METRIC_GROUP_NAME, "The 99th percentile latency of committing offset", tags), 99.0),
      new Percentile(new MetricName("commit-offset-latency-ms-999th", METRIC_GROUP_NAME, "The 99.9th percentile latency of committing offset", tags), 99.9),
      new Percentile(new MetricName("commit-offset-latency-ms-9999th", METRIC_GROUP_NAME, "The 99.99th percentile latency of committing offset", tags), 99.99)));
  LOG.info("{} was constructed successfully.", this.getClass().getSimpleName());
}
 

public KsqlEngineMetrics(String metricGroupPrefix, KsqlEngine ksqlEngine) {
  this.ksqlEngine = ksqlEngine;
  this.sensors = new ArrayList<>();
  this.metricGroupName = metricGroupPrefix + "-query-stats";

  Metrics metrics = MetricCollectors.getMetrics();

  this.numActiveQueries = configureNumActiveQueries(metrics);
  this.messagesIn = configureMessagesIn(metrics);
  this.totalMessagesIn = configureTotalMessagesIn(metrics);
  this.totalBytesIn = configureTotalBytesIn(metrics);
  this.messagesOut =  configureMessagesOut(metrics);
  this.numIdleQueries = configureIdleQueriesSensor(metrics);
  this.messageConsumptionByQuery = configureMessageConsumptionByQuerySensor(metrics);
  this.errorRate = configureErrorRate(metrics);
}
 

@Test
public void testMetricChange() throws Exception {
    Metrics metrics = new Metrics();
    DropwizardReporter reporter = new DropwizardReporter();
    reporter.configure(new HashMap<String, Object>());
    metrics.addReporter(reporter);
    Sensor sensor = metrics.sensor("kafka.requests");
    sensor.add(new MetricName("pack.bean1.avg", "grp1"), new Avg());

    Map<String, Gauge> gauges = SharedMetricRegistries.getOrCreate("default").getGauges();
    String expectedName = "org.apache.kafka.common.metrics.grp1.pack.bean1.avg";
    Assert.assertEquals(1, gauges.size());
    Assert.assertEquals(expectedName, gauges.keySet().toArray()[0]);

    sensor.record(2.1);
    sensor.record(2.2);
    sensor.record(2.6);
    Assert.assertEquals(2.3, (Double)gauges.get(expectedName).getValue(), 0.001);
}
 

public KafkaNodeClient(int id, String host, int port) {
	node = new Node(id, host, port);
	
	//
	LogContext logContext = new LogContext("ctx");

	ConfigDef defConf = new ConfigDef();
	defConf.define(CommonClientConfigs.SECURITY_PROTOCOL_CONFIG, ConfigDef.Type.STRING,
			CommonClientConfigs.DEFAULT_SECURITY_PROTOCOL, ConfigDef.Importance.MEDIUM,
			CommonClientConfigs.SECURITY_PROTOCOL_DOC);

	defConf.define(SaslConfigs.SASL_MECHANISM, ConfigDef.Type.STRING, SaslConfigs.DEFAULT_SASL_MECHANISM,
			ConfigDef.Importance.MEDIUM, SaslConfigs.SASL_MECHANISM_DOC);

	metrics = new Metrics(Time.SYSTEM);

	AbstractConfig config = new AbstractConfig(defConf, new Properties());
	channelBuilder = ClientUtils.createChannelBuilder(config);
	selector = new Selector(1000L, metrics, Time.SYSTEM, "cc", channelBuilder, logContext);
	client = new NetworkClient(selector, new Metadata(0, Long.MAX_VALUE, false),
			CLIENT_ID, 10, 1000L, 1000L, 1, 1024, 1000, Time.SYSTEM, true, new ApiVersions(),
			null, logContext);
}
 

@Override
public NetworkClient createNetworkClient(long connectionMaxIdleMS,
                                         Metrics metrics,
                                         Time time,
                                         String metricGrpPrefix,
                                         ChannelBuilder channelBuilder,
                                         Metadata metadata,
                                         String clientId,
                                         int maxInFlightRequestsPerConnection,
                                         long reconnectBackoffMs,
                                         long reconnectBackoffMax,
                                         int socketSendBuffer,
                                         int socketReceiveBuffer,
                                         int defaultRequestTimeoutMs,
                                         boolean discoverBrokerVersions,
                                         ApiVersions apiVersions) {
  return new NetworkClient(new Selector(connectionMaxIdleMS, metrics, time, metricGrpPrefix, channelBuilder, new LogContext()),
                           metadata, clientId, maxInFlightRequestsPerConnection, reconnectBackoffMs,
                           reconnectBackoffMax, socketSendBuffer, socketReceiveBuffer, defaultRequestTimeoutMs,
                           ClientDnsLookup.DEFAULT, time, discoverBrokerVersions, apiVersions, new LogContext());
}
 

MissingPartitionsJmxReporter(Metrics metrics) {
  super(metrics);
  Sensor missingPartsSensor = metrics.sensor(MISSING_DEST_PARTITIONS);
  MetricName missingPartsName = metrics.metricName(MISSING_DEST_PARTITIONS + "-count", "mirus");
  missingPartsSensor.add(missingPartsName, new Value());
  this.missingPartsSensor = missingPartsSensor;
}
 

private Sensor configureErrorRate(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-error-rate");
  sensor.add(
      metrics.metricName("error-rate", this.metricGroupName,
                         "The number of messages which were consumed but not processed. "
                         + "Messages may not be processed if, for instance, the message "
                         + "contents could not be deserialized due to an incompatible schema. "
                         + "Alternately, a consumed messages may not have been produced, hence "
                         + "being effectively dropped. Such messages would also be counted "
                         + "toward the error rate."),
      new Value());
  return sensor;
}
 

private Sensor configureMessagesOut(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-messages-produced");
  sensor.add(
      metrics.metricName("messages-produced-per-sec", this.metricGroupName,
                         "The number of messages produced per second across all queries"),
      new Value());

  return sensor;
}
 

private Sensor configureTotalMessagesIn(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-total-messages-consumed");
  sensor.add(
      metrics.metricName("messages-consumed-total", this.metricGroupName,
          "The total number of messages consumed across all queries"),
      new Value());
  return sensor;
}
 

private Sensor configureTotalBytesIn(Metrics metrics) {
  Sensor sensor = createSensor(metrics, metricGroupName + "-total-bytes-consumed");
  sensor.add(
      metrics.metricName("bytes-consumed-total", this.metricGroupName,
          "The total number of bytes consumed across all queries"),
      new Value());
  return sensor;
}
 

private Sensor configureMessageConsumptionByQuerySensor(Metrics metrics) {
  Sensor sensor = createSensor(metrics, "message-consumption-by-query");
  sensor.add(metrics.metricName("messages-consumed-max", this.metricGroupName), new Max());
  sensor.add(metrics.metricName("messages-consumed-min", this.metricGroupName), new Min());
  sensor.add(metrics.metricName("messages-consumed-avg", this.metricGroupName), new Avg());
  return sensor;
}
 

@Test
public void shouldRemoveAllSensorsOnClose() {
  assertTrue(engineMetrics.registeredSensors().size() > 0);

  engineMetrics.close();

  Metrics metrics = MetricCollectors.getMetrics();
  engineMetrics.registeredSensors().forEach(sensor -> {
    assertTrue(metrics.getSensor(sensor.name()) == null);
  });
}
 

public MetricsResourceMethodApplicationListener(Metrics metrics, String metricGrpPrefix,
                                         Map<String,String> metricTags, Time time) {
  super();
  this.metrics = metrics;
  this.metricGrpPrefix = metricGrpPrefix;
  this.metricTags = (metricTags != null) ? metricTags : Collections.<String,String>emptyMap();
  this.time = time;
}
 

/**
 * XinfraMonitor constructor creates apps and services for each of the individual clusters (properties) that's passed in.
 * For example, if there are 10 clusters to be monitored, then this Constructor will create 10 * num_apps_per_cluster
 * and 10 * num_services_per_cluster.
 * @param allClusterProps the properties of ALL kafka clusters for which apps and services need to be appended.
 * @throws Exception when exception occurs while assigning Apps and Services
 */

@SuppressWarnings({"rawtypes"})
public XinfraMonitor(Map<String, Map> allClusterProps) throws Exception {
  _apps = new ConcurrentHashMap<>();
  _services = new ConcurrentHashMap<>();

  for (Map.Entry<String, Map> clusterProperty : allClusterProps.entrySet()) {
    String name = clusterProperty.getKey();
    Map props = clusterProperty.getValue();
    if (!props.containsKey(XinfraMonitorConstants.CLASS_NAME_CONFIG))
      throw new IllegalArgumentException(name + " is not configured with " + XinfraMonitorConstants.CLASS_NAME_CONFIG);
    String className = (String) props.get(XinfraMonitorConstants.CLASS_NAME_CONFIG);

    Class<?> aClass = Class.forName(className);
    if (App.class.isAssignableFrom(aClass)) {
      App clusterApp = (App) Class.forName(className).getConstructor(Map.class, String.class).newInstance(props, name);
      _apps.put(name, clusterApp);
    } else if (Service.class.isAssignableFrom(aClass)) {
      ServiceFactory serviceFactory = (ServiceFactory) Class.forName(className + XinfraMonitorConstants.FACTORY)
          .getConstructor(Map.class, String.class)
          .newInstance(props, name);
      Service service = serviceFactory.createService();
      _services.put(name, service);
    } else {
      throw new IllegalArgumentException(className + " should implement either " + App.class.getSimpleName() + " or " + Service.class.getSimpleName());
    }
  }
  _executor = Executors.newSingleThreadScheduledExecutor();
  _offlineRunnables = new ConcurrentHashMap<>();
  List<MetricsReporter> reporters = new ArrayList<>();
  reporters.add(new JmxReporter(XinfraMonitorConstants.JMX_PREFIX));
  Metrics metrics = new Metrics(new MetricConfig(), reporters, new SystemTime());
  metrics.addMetric(metrics.metricName("offline-runnable-count", XinfraMonitorConstants.METRIC_GROUP_NAME, "The number of Service/App that are not fully running"),
    (config, now) -> _offlineRunnables.size());
}
 

@Before
public void setUp() throws Exception {
  metrics = new Metrics();
  connectorInfo =
      new ConnectorInfo(CONNECTOR_NAME, new HashMap<>(), new ArrayList<>(), ConnectorType.SOURCE);
  tags = new HashMap<>();
  tags.put("connector", CONNECTOR_NAME);

  connectorJmxReporter = new ConnectorJmxReporter(metrics);
  connectorJmxReporter.handleConnector(herder, connectorInfo);
}
 

@Test
public void shouldRecordErrors() throws Exception {

  ProducerCollector collector = new ProducerCollector().configure(new Metrics(), "clientid", MetricCollectors.getTime());

  for (int i = 0; i < 1000; i++){
    collector.recordError(TEST_TOPIC);
  }

  Collection<TopicSensors.Stat> stats = collector.stats("test-topic", true);

  assertThat( stats.toString(), containsString("failed-messages"));
  assertThat( stats.toString(), containsString("value=1000"));
}
 

/**
 * Initialize the coordination manager.
 */
public WorkerCoordinator(ConsumerNetworkClient client,
                         String groupId,
                         int rebalanceTimeoutMs,
                         int sessionTimeoutMs,
                         int heartbeatIntervalMs,
                         Metrics metrics,
                         String metricGrpPrefix,
                         Time time,
                         long retryBackoffMs,
                         String restUrl,
                         TaskConfigManager taskConfigManager,
                         WorkerRebalanceListener listener) {
    super(client,
            groupId,
            rebalanceTimeoutMs,
            sessionTimeoutMs,
            heartbeatIntervalMs,
            metrics,
            metricGrpPrefix,
            time,
            retryBackoffMs);
    this.restUrl = restUrl;
    this.taskConfigManager = taskConfigManager;
    this.assignmentSnapshot = null;
    this.sensors = new WorkerCoordinatorMetrics(metrics, metricGrpPrefix);
    this.listener = listener;
    this.rejoinRequested = false;
}
 

public WorkerCoordinatorMetrics(Metrics metrics, String metricGrpPrefix) {
    this.metrics = metrics;
    this.metricGrpName = metricGrpPrefix + "-coordinator-metrics";

    Measurable numTasks = (config, now) -> assignmentSnapshot.tasks().size();
    metrics.addMetric(metrics.metricName("assigned-tasks",
            this.metricGrpName,
            "The number of tasks currently assigned to this consumer"), numTasks);
}
 

public MetadataClient(KafkaCruiseControlConfig config,
                      Metadata metadata,
                      long metadataTTL,
                      Time time) {
  _metadataGeneration = new AtomicInteger(0);
  _metadata = metadata;
  _refreshMetadataTimeout = config.getLong(MonitorConfig.METADATA_MAX_AGE_CONFIG);
  _time = time;
  List<InetSocketAddress> addresses =
      ClientUtils.parseAndValidateAddresses(config.getList(MonitorConfig.BOOTSTRAP_SERVERS_CONFIG),
                                            ClientDnsLookup.DEFAULT);
  Cluster bootstrapCluster = Cluster.bootstrap(addresses);
  MetadataResponse metadataResponse = KafkaCruiseControlUtils.prepareMetadataResponse(bootstrapCluster.nodes(),
                                                                                      bootstrapCluster.clusterResource().clusterId(),
                                                                                      MetadataResponse.NO_CONTROLLER_ID,
                                                                                      Collections.emptyList());

  _metadata.update(KafkaCruiseControlUtils.REQUEST_VERSION_UPDATE, metadataResponse, time.milliseconds());
  ChannelBuilder channelBuilder = ClientUtils.createChannelBuilder(config, time);
  NetworkClientProvider provider = config.getConfiguredInstance(MonitorConfig.NETWORK_CLIENT_PROVIDER_CLASS_CONFIG,
                                                                NetworkClientProvider.class);

  _networkClient = provider.createNetworkClient(config.getLong(MonitorConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG),
                                                new Metrics(),
                                                time,
                                                "load-monitor",
                                                channelBuilder,
                                                _metadata,
                                                config.getString(MonitorConfig.CLIENT_ID_CONFIG),
                                                DEFAULT_MAX_IN_FLIGHT_REQUEST,
                                                config.getLong(MonitorConfig.RECONNECT_BACKOFF_MS_CONFIG),
                                                config.getLong(MonitorConfig.RECONNECT_BACKOFF_MS_CONFIG),
                                                config.getInt(MonitorConfig.SEND_BUFFER_CONFIG),
                                                config.getInt(MonitorConfig.RECEIVE_BUFFER_CONFIG),
                                                config.getInt(MonitorConfig.REQUEST_TIMEOUT_MS_CONFIG),
                                                true,
                                                new ApiVersions());
  _metadataTTL = metadataTTL;
}
 

/**
 * Initialize the coordination manager.
 */
public KarelDbCoordinator(
    LogContext logContext,
    ConsumerNetworkClient client,
    String groupId,
    int rebalanceTimeoutMs,
    int sessionTimeoutMs,
    int heartbeatIntervalMs,
    Metrics metrics,
    String metricGrpPrefix,
    Time time,
    long retryBackoffMs,
    KarelDbIdentity identity,
    KarelDbRebalanceListener listener) {
    super(
        new GroupRebalanceConfig(
            sessionTimeoutMs,
            rebalanceTimeoutMs,
            heartbeatIntervalMs,
            groupId,
            Optional.empty(),
            retryBackoffMs,
            true
        ),
        logContext,
        client,
        metrics,
        metricGrpPrefix,
        time
    );
    this.identity = identity;
    this.assignmentSnapshot = null;
    this.listener = listener;
}
 

ConnectorJmxReporter(Metrics metrics) {
  super(metrics);
  allStates.put("RUNNING", "running");
  allStates.put("FAILED", "failed");
  allStates.put("DESTROYED", "destroyed");
  allStates.put("UNASSIGNED", "unassigned");
  allStates.put("PAUSED", "paused");
  connectorLevelJmxTags.add(CONNECTOR_KEY);
}
 

/**
 * Mainly contains services for three metrics:
 * 1 - ConsumeAvailability metrics
 * 2 - CommitOffsetAvailability metrics
 *   2.1 - commitAvailabilityMetrics records offsets committed upon success. that is, no exception upon callback
 *   2.2 - commitAvailabilityMetrics records offsets commit fail upon failure. that is, exception upon callback
 * 3 - CommitOffsetLatency metrics
 *   3.1 - commitLatencyMetrics records the latency between last successful callback and start of last recorded commit.
 *
 * @param name Name of the Monitor instance
 * @param topicPartitionResult The completable future for topic partition
 * @param consumerFactory Consumer Factory object.
 * @throws ExecutionException when attempting to retrieve the result of a task that aborted by throwing an exception
 * @throws InterruptedException when a thread is waiting, sleeping, or otherwise occupied and the thread is interrupted
 */
public ConsumeService(String name,
                      CompletableFuture<Void> topicPartitionResult,
                      ConsumerFactory consumerFactory)
    throws ExecutionException, InterruptedException {
  _baseConsumer = consumerFactory.baseConsumer();
  _latencySlaMs = consumerFactory.latencySlaMs();
  _name = name;
  _adminClient = consumerFactory.adminClient();
  _running = new AtomicBoolean(false);

  // Returns a new CompletionStage (topicPartitionFuture) which
  // executes the given action - code inside run() - when this stage (topicPartitionResult) completes normally,.
  CompletableFuture<Void> topicPartitionFuture = topicPartitionResult.thenRun(() -> {
    MetricConfig metricConfig = new MetricConfig().samples(60).timeWindow(1000, TimeUnit.MILLISECONDS);
    List<MetricsReporter> reporters = new ArrayList<>();
    reporters.add(new JmxReporter(JMX_PREFIX));
    metrics = new Metrics(metricConfig, reporters, new SystemTime());
    tags = new HashMap<>();
    tags.put(TAGS_NAME, name);
    _topic = consumerFactory.topic();
    _sensors = new ConsumeMetrics(metrics, tags, consumerFactory.latencyPercentileMaxMs(),
        consumerFactory.latencyPercentileGranularityMs());
    _commitLatencyMetrics = new CommitLatencyMetrics(metrics, tags, consumerFactory.latencyPercentileMaxMs(),
        consumerFactory.latencyPercentileGranularityMs());
    _commitAvailabilityMetrics = new CommitAvailabilityMetrics(metrics, tags);
    _consumeThread = new Thread(() -> {
      try {
        consume();
      } catch (Exception e) {
        LOG.error(name + "/ConsumeService failed", e);
      }
    }, name + " consume-service");
    _consumeThread.setDaemon(true);
  });

  // In a blocking fashion, waits for this topicPartitionFuture to complete, and then returns its result.
  topicPartitionFuture.get();
}