org.apache.kafka.connect.runtime.distributed.DistributedConfig Java Examples

static DistributedHerder createDistributedHerder(DistributedConfig config,
                                                 Time time,
                                                 Worker worker,
                                                 String kafkaClusterId,
                                                 StatusBackingStore statusBackingStore,
                                                 ConfigBackingStore configBackingStore,
                                                 String restUrl,
                                                 Object connectorClientConfigOverridePolicy) throws ConnectException {

    if (CTR_DISTRIBUTED_HERDER_22 == null) {
        return new DistributedHerder(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, restUrl, (ConnectorClientConfigOverridePolicy)connectorClientConfigOverridePolicy);
    }
    try {
        return (DistributedHerder)CTR_DISTRIBUTED_HERDER_22.newInstance(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, restUrl);
    } catch (Throwable t) {
        throw new ConnectException(t);
    }
}
 

private Properties connectWorkerConfig() {
  Properties workerProps = new Properties();
  workerProps.put(DistributedConfig.GROUP_ID_CONFIG, "mongo-kafka-test");
  workerProps.put(DistributedConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers());
  workerProps.put(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, "connect-offsets");
  workerProps.put(DistributedConfig.CONFIG_TOPIC_CONFIG, "connect-configs");
  workerProps.put(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG, "connect-status");
  workerProps.put(
      DistributedConfig.KEY_CONVERTER_CLASS_CONFIG,
      "org.apache.kafka.connect.storage.StringConverter");
  workerProps.put("key.converter.schemas.enable", "false");
  workerProps.put(
      DistributedConfig.VALUE_CONVERTER_CLASS_CONFIG,
      "org.apache.kafka.connect.storage.StringConverter");
  workerProps.put("value.converter.schemas.enable", "false");
  workerProps.put(DistributedConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "100");
  workerProps.put(
      StandaloneConfig.OFFSET_STORAGE_FILE_FILENAME_CONFIG,
      createTempDirectory().getAbsolutePath() + "connect");

  return workerProps;
}
 

private static MirusOffsetTool newOffsetTool(Args args) throws IOException {
  // This needs to be the admin topic properties.
  // By default these are in the worker properties file, as this has the has admin producer and
  // consumer settings.  Separating these might be wise - also useful for storing state in
  // source cluster if it proves necessary.
  final Map<String, String> properties =
      !args.propertiesFile.isEmpty()
          ? Utils.propsToStringMap(Utils.loadProps(args.propertiesFile))
          : Collections.emptyMap();
  final DistributedConfig config = new DistributedConfig(properties);
  final KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
  offsetBackingStore.configure(config);

  // Avoid initializing the entire Kafka Connect plugin system by assuming the
  // internal.[key|value].converter is org.apache.kafka.connect.json.JsonConverter
  final Converter internalConverter = new JsonConverter();
  internalConverter.configure(config.originalsWithPrefix("internal.key.converter."), true);

  final OffsetSetter offsetSetter = new OffsetSetter(internalConverter, offsetBackingStore);
  final OffsetFetcher offsetFetcher = new OffsetFetcher(config, internalConverter);
  final OffsetSerDe offsetSerDe = OffsetSerDeFactory.create(args.format);

  return new MirusOffsetTool(args, offsetFetcher, offsetSetter, offsetSerDe);
}
 

public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception {
    Time time = new SystemTime();
    DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig));

    KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
    offsetBackingStore.configure(config);

    //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random
    String workerId = UUID.randomUUID().toString();
    worker = new Worker(workerId, time, config, offsetBackingStore);

    StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter());
    statusBackingStore.configure(config);

    ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter());
    configBackingStore.configure(config);

    //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything
    herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, "");
    this.connectorConfigs = connectorConfigs;

    shutdownHook = new ShutdownHook();
}
 

OffsetFetcher(final WorkerConfig config, Converter internalConverter) {
  String topic = config.getString(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG);
  if ("".equals(topic)) {
    throw new ConfigException("Offset storage topic must be specified");
  }

  Map<String, Object> producerProps = new HashMap<>(config.originals());
  producerProps.put(
      ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
  producerProps.put(
      ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName());
  producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE);

  Map<String, Object> consumerProps = new HashMap<>(config.originals());
  consumerProps.put(
      ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());
  consumerProps.put(
      ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName());

  Callback<ConsumerRecord<byte[], byte[]>> consumedCallback =
      (error, record) -> {
        ByteBuffer key = record.key() != null ? ByteBuffer.wrap(record.key()) : null;
        ByteBuffer value = record.value() != null ? ByteBuffer.wrap(record.value()) : null;
        data.put(key, value);
      };
  this.offsetLog =
      new KafkaBasedLog<>(
          topic, producerProps, consumerProps, consumedCallback, Time.SYSTEM, null);
  this.internalConverter = internalConverter;
}
 

/**
 * Create a new DistributedConfig object with a suffix applied to the client id. This allows us to
 * make the client id unique so JMX metrics work properly.
 */
private static DistributedConfig configWithClientIdSuffix(
    Map<String, String> workerProps, String suffix) {
  Map<String, String> localProps = new HashMap<>(workerProps);
  localProps.computeIfPresent(CommonClientConfigs.CLIENT_ID_CONFIG, (k, v) -> v + suffix);
  return new DistributedConfig(localProps);
}
 

private static ConnectEmbedded createWikipediaFeedConnectInstance(String bootstrapServers) throws Exception {
    Properties workerProps = new Properties();
    workerProps.put(DistributedConfig.GROUP_ID_CONFIG, "wikipedia-connect");
    workerProps.put(DistributedConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
    workerProps.put(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, "connect-offsets");
    workerProps.put(DistributedConfig.CONFIG_TOPIC_CONFIG, "connect-configs");
    workerProps.put(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG, "connect-status");
    workerProps.put(DistributedConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("key.converter.schemas.enable", "false");
    workerProps.put(DistributedConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("value.converter.schemas.enable", "false");
    workerProps.put(DistributedConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "30000");
    workerProps.put(DistributedConfig.INTERNAL_KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter.schemas.enable", "false");
    workerProps.put(DistributedConfig.INTERNAL_VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.value.converter.schemas.enable", "false");

    Properties connectorProps = new Properties();
    connectorProps.put(ConnectorConfig.NAME_CONFIG, "wikipedia-irc-source");
    connectorProps.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, "io.amient.kafka.connect.irc.IRCFeedConnector");
    connectorProps.put(ConnectorConfig.TASKS_MAX_CONFIG, "10");
    connectorProps.put(IRCFeedConnector.IRC_HOST_CONFIG, "irc.wikimedia.org");
    connectorProps.put(IRCFeedConnector.IRC_PORT_CONFIG, "6667");
    connectorProps.put(IRCFeedConnector.IRC_CHANNELS_CONFIG, "#en.wikipedia,#en.wiktionary,#en.wikinews");
    connectorProps.put(IRCFeedConnector.TOPIC_CONFIG, "wikipedia-raw");

    return new ConnectEmbedded(workerProps, connectorProps);

}
 

private Connect startConnect(Map<String, String> workerProps) {
    log.info("Scanning for plugin classes. This might take a moment ...");
    Plugins plugins = new Plugins(workerProps);
    // ignore this TCCL switch plugins.compareAndSwapWithDelegatingLoader();
    DistributedConfig config = new DistributedConfig(workerProps);

    String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
    log.debug("Kafka cluster ID: {}", kafkaClusterId);

    RestServer rest = new RestServer(config);
    rest.initializeServer();

    URI advertisedUrl = rest.advertisedUrl();
    String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();

    KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
    offsetBackingStore.configure(config);
    Object connectorClientConfigOverridePolicy = Compatibility.createConnectorClientConfigOverridePolicy(plugins, config);

    Worker worker = Compatibility.createWorker(workerId, time, plugins, config, offsetBackingStore, connectorClientConfigOverridePolicy);
    WorkerConfigTransformer configTransformer = worker.configTransformer();

    Converter internalValueConverter = worker.getInternalValueConverter();
    StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
    statusBackingStore.configure(config);

    ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(
        internalValueConverter,
        config,
        configTransformer);

    DistributedHerder herder = Compatibility.createDistributedHerder(config, time, worker,
                                                     kafkaClusterId, statusBackingStore, configBackingStore,
                                                     advertisedUrl.toString(), connectorClientConfigOverridePolicy);

    final Connect connect = new Connect(herder, rest);
    log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart);
    try {
        connect.start();
    } catch (Exception e) {
        log.error("Failed to start Connect", e);
        connect.stop();
        throw new IllegalStateException(e);
    }

    return connect;
}
 

/**
 * This method is based on the the standard Kafka Connect start logic in {@link
 * org.apache.kafka.connect.cli.ConnectDistributed#startConnect(Map)}, but with `clientid` prefix
 * support, to prevent JMX metric names from clashing. Also supports command-line property
 * overrides (useful for run-time port configuration), and starts the Mirus {@link
 * HerderStatusMonitor}.
 */
public Connect startConnect(Map<String, String> workerProps) {
  log.info("Scanning for plugin classes. This might take a moment ...");
  Plugins plugins = new Plugins(workerProps);
  plugins.compareAndSwapWithDelegatingLoader();
  DistributedConfig distributedConfig = configWithClientIdSuffix(workerProps, "herder");

  MirusConfig mirusConfig = new MirusConfig(workerProps);

  String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(distributedConfig);
  log.debug("Kafka cluster ID: {}", kafkaClusterId);

  RestServer rest = new RestServer(configWithClientIdSuffix(workerProps, "rest"));
  rest.initializeServer();

  URI advertisedUrl = rest.advertisedUrl();
  String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();

  KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
  offsetBackingStore.configure(configWithClientIdSuffix(workerProps, "offset"));

  WorkerConfig workerConfigs = configWithClientIdSuffix(workerProps, "worker");

  ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy =
      plugins.newPlugin(
          distributedConfig.getString(WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG),
          workerConfigs,
          ConnectorClientConfigOverridePolicy.class);

  Worker worker =
      new Worker(
          workerId,
          time,
          plugins,
          workerConfigs,
          offsetBackingStore,
          connectorClientConfigOverridePolicy);

  WorkerConfigTransformer configTransformer = worker.configTransformer();

  Converter internalValueConverter = worker.getInternalValueConverter();
  StatusBackingStore statusBackingStore =
      new KafkaStatusBackingStore(time, internalValueConverter);
  statusBackingStore.configure(configWithClientIdSuffix(workerProps, "status"));

  ConfigBackingStore configBackingStore =
      new KafkaConfigBackingStore(
          internalValueConverter,
          configWithClientIdSuffix(workerProps, "config"),
          configTransformer);

  DistributedHerder herder =
      new DistributedHerder(
          distributedConfig,
          time,
          worker,
          kafkaClusterId,
          statusBackingStore,
          configBackingStore,
          advertisedUrl.toString(),
          connectorClientConfigOverridePolicy);

  // Initialize HerderStatusMonitor
  boolean autoStartTasks = mirusConfig.getTaskAutoRestart();
  boolean autoStartConnectors = mirusConfig.getConnectorAutoRestart();
  long pollingCycle = mirusConfig.getTaskStatePollingInterval();
  HerderStatusMonitor herderStatusMonitor =
      new HerderStatusMonitor(
          herder, workerId, pollingCycle, autoStartTasks, autoStartConnectors);
  Thread herderStatusMonitorThread = new Thread(herderStatusMonitor);
  herderStatusMonitorThread.setName("herder-status-monitor");

  final Connect connect = new Connect(herder, rest);
  log.info("Mirus worker initialization took {}ms", time.hiResClockMs() - initStart);
  try {
    connect.start();
  } catch (Exception e) {
    log.error("Failed to start Mirus", e);
    connect.stop();
    Exit.exit(3);
  }

  herderStatusMonitorThread.start();

  return connect;
}