org.apache.kafka.connect.runtime.Worker Java Examples
The following examples show how to use
org.apache.kafka.connect.runtime.Worker.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Compatibility.java From apicurio-registry with Apache License 2.0 | 6 votes |
static Worker createWorker(String workerId, Time time, Plugins plugins, WorkerConfig config, OffsetBackingStore offsetBackingStore, Object connectorClientConfigOverridePolicy) throws ConnectException { if (CTR_WORKER_22 == null) { return new Worker(workerId, time, plugins, config, offsetBackingStore, (ConnectorClientConfigOverridePolicy)connectorClientConfigOverridePolicy); } try { return (Worker)CTR_WORKER_22.newInstance(workerId, time, plugins, config, offsetBackingStore); } catch (Throwable t) { throw new ConnectException(t); } }
Example #2
Source File: Compatibility.java From apicurio-registry with Apache License 2.0 | 6 votes |
static DistributedHerder createDistributedHerder(DistributedConfig config, Time time, Worker worker, String kafkaClusterId, StatusBackingStore statusBackingStore, ConfigBackingStore configBackingStore, String restUrl, Object connectorClientConfigOverridePolicy) throws ConnectException { if (CTR_DISTRIBUTED_HERDER_22 == null) { return new DistributedHerder(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, restUrl, (ConnectorClientConfigOverridePolicy)connectorClientConfigOverridePolicy); } try { return (DistributedHerder)CTR_DISTRIBUTED_HERDER_22.newInstance(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, restUrl); } catch (Throwable t) { throw new ConnectException(t); } }
Example #3
Source File: ConnectEmbedded.java From hello-kafka-streams with Apache License 2.0 | 6 votes |
public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception { Time time = new SystemTime(); DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig)); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config); //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random String workerId = UUID.randomUUID().toString(); worker = new Worker(workerId, time, config, offsetBackingStore); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter()); statusBackingStore.configure(config); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter()); configBackingStore.configure(config); //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, ""); this.connectorConfigs = connectorConfigs; shutdownHook = new ShutdownHook(); }
Example #4
Source File: IgniteSourceConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { kafkaBroker = new TestKafkaBroker(); Map<String, String> props = makeWorkerProps(); WorkerConfig workerCfg = new StandaloneConfig(props); MemoryOffsetBackingStore offBackingStore = new MemoryOffsetBackingStore(); offBackingStore.configure(workerCfg); worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore); worker.start(); herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg)); herder.start(); }
Example #5
Source File: IgniteSinkConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { kafkaBroker = new TestKafkaBroker(); for (String topic : TOPICS) kafkaBroker.createTopic(topic, PARTITIONS, REPLICATION_FACTOR); Map<String, String> props = makeWorkerProps(); WorkerConfig workerCfg = new StandaloneConfig(props); OffsetBackingStore offBackingStore = mock(OffsetBackingStore.class); offBackingStore.configure(workerCfg); worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore); worker.start(); herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg)); herder.start(); }
Example #6
Source File: KafkaConnectRunner.java From camel-kafka-connector with Apache License 2.0 | 5 votes |
/** * here does not seem to be a public interface for embedding a Kafka connect runtime, * therefore, this code is modeled from the behavior taken from * https://github.com/apache/kafka/blob/2.1/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java * and performs the initialization in a roughly similar manner. * */ private void init() { LOG.info("Started worked initialization"); Time time = Time.SYSTEM; // Initializes the system runtime information and logs some of the information WorkerInfo initInfo = new WorkerInfo(); initInfo.logAll(); Properties props = kafkaConnectPropertyFactory.getProperties(); Map<String, String> standAloneProperties = Utils.propsToStringMap(props); // Not needed, but we need this one to initialize the worker Plugins plugins = new Plugins(standAloneProperties); StandaloneConfig config = new StandaloneConfig(standAloneProperties); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config); AllConnectorClientConfigOverridePolicy allConnectorClientConfigOverridePolicy = new AllConnectorClientConfigOverridePolicy(); RestServer rest = new RestServer(config); rest.initializeServer(); /* According to the Kafka source code "... Worker runs a (dynamic) set of tasks in a set of threads, doing the work of actually moving data to/from Kafka ..." */ Worker worker = new Worker(bootstrapServer, time, plugins, config, new FileOffsetBackingStore(), allConnectorClientConfigOverridePolicy); /* From Kafka source code: " ... The herder interface tracks and manages workers and connectors ..." */ herder = new StandaloneHerder(worker, kafkaClusterId, allConnectorClientConfigOverridePolicy); connect = new Connect(herder, rest); LOG.info("Finished initializing the worker"); }
Example #7
Source File: ConnectRunner.java From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0 | 5 votes |
void start() { final Map<String, String> workerProps = new HashMap<>(); workerProps.put("bootstrap.servers", bootstrapServers); workerProps.put("offset.flush.interval.ms", Integer.toString(offsetFlushInterval)); // These don't matter much (each connector sets its own converters), but need to be filled with valid classes. workerProps.put("key.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); workerProps.put("value.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter"); workerProps.put("internal.key.converter.schemas.enable", "false"); workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter"); workerProps.put("internal.value.converter.schemas.enable", "false"); // Don't need it since we'll memory MemoryOffsetBackingStore. workerProps.put("offset.storage.file.filename", ""); workerProps.put("plugin.path", pluginDir.getPath()); final Time time = Time.SYSTEM; final String workerId = "test-worker"; final Plugins plugins = new Plugins(workerProps); final StandaloneConfig config = new StandaloneConfig(workerProps); final Worker worker = new Worker( workerId, time, plugins, config, new MemoryOffsetBackingStore()); herder = new StandaloneHerder(worker); final RestServer rest = new RestServer(config); connect = new Connect(herder, rest); connect.start(); }
Example #8
Source File: ConnectStandalone.java From mongo-kafka with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") ConnectStandalone(final Properties workerProperties) { Time time = Time.SYSTEM; LOGGER.info("Kafka Connect standalone worker initializing ..."); long initStart = time.hiResClockMs(); WorkerInfo initInfo = new WorkerInfo(); initInfo.logAll(); Map<String, String> workerProps = (Map) workerProperties; LOGGER.info("Scanning for plugin classes. This might take a moment ..."); Plugins plugins = new Plugins(workerProps); plugins.compareAndSwapWithDelegatingLoader(); StandaloneConfig config = new StandaloneConfig(workerProps); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config); LOGGER.debug("Kafka cluster ID: {}", kafkaClusterId); RestServer rest = new RestServer(config); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore()); this.herder = new StandaloneHerder(worker, kafkaClusterId); connectionString = advertisedUrl.toString() + herder.kafkaClusterId(); this.connect = new Connect(herder, rest); LOGGER.info( "Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart); }
Example #9
Source File: ConnectorApplication.java From apicurio-registry with Apache License 2.0 | 4 votes |
private Connect startConnect(Map<String, String> workerProps) { log.info("Scanning for plugin classes. This might take a moment ..."); Plugins plugins = new Plugins(workerProps); // ignore this TCCL switch plugins.compareAndSwapWithDelegatingLoader(); DistributedConfig config = new DistributedConfig(workerProps); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config); log.debug("Kafka cluster ID: {}", kafkaClusterId); RestServer rest = new RestServer(config); rest.initializeServer(); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config); Object connectorClientConfigOverridePolicy = Compatibility.createConnectorClientConfigOverridePolicy(plugins, config); Worker worker = Compatibility.createWorker(workerId, time, plugins, config, offsetBackingStore, connectorClientConfigOverridePolicy); WorkerConfigTransformer configTransformer = worker.configTransformer(); Converter internalValueConverter = worker.getInternalValueConverter(); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter); statusBackingStore.configure(config); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore( internalValueConverter, config, configTransformer); DistributedHerder herder = Compatibility.createDistributedHerder(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, advertisedUrl.toString(), connectorClientConfigOverridePolicy); final Connect connect = new Connect(herder, rest); log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart); try { connect.start(); } catch (Exception e) { log.error("Failed to start Connect", e); connect.stop(); throw new IllegalStateException(e); } return connect; }
Example #10
Source File: Mirus.java From mirus with BSD 3-Clause "New" or "Revised" License | 4 votes |
/** * This method is based on the the standard Kafka Connect start logic in {@link * org.apache.kafka.connect.cli.ConnectDistributed#startConnect(Map)}, but with `clientid` prefix * support, to prevent JMX metric names from clashing. Also supports command-line property * overrides (useful for run-time port configuration), and starts the Mirus {@link * HerderStatusMonitor}. */ public Connect startConnect(Map<String, String> workerProps) { log.info("Scanning for plugin classes. This might take a moment ..."); Plugins plugins = new Plugins(workerProps); plugins.compareAndSwapWithDelegatingLoader(); DistributedConfig distributedConfig = configWithClientIdSuffix(workerProps, "herder"); MirusConfig mirusConfig = new MirusConfig(workerProps); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(distributedConfig); log.debug("Kafka cluster ID: {}", kafkaClusterId); RestServer rest = new RestServer(configWithClientIdSuffix(workerProps, "rest")); rest.initializeServer(); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(configWithClientIdSuffix(workerProps, "offset")); WorkerConfig workerConfigs = configWithClientIdSuffix(workerProps, "worker"); ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy = plugins.newPlugin( distributedConfig.getString(WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG), workerConfigs, ConnectorClientConfigOverridePolicy.class); Worker worker = new Worker( workerId, time, plugins, workerConfigs, offsetBackingStore, connectorClientConfigOverridePolicy); WorkerConfigTransformer configTransformer = worker.configTransformer(); Converter internalValueConverter = worker.getInternalValueConverter(); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter); statusBackingStore.configure(configWithClientIdSuffix(workerProps, "status")); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore( internalValueConverter, configWithClientIdSuffix(workerProps, "config"), configTransformer); DistributedHerder herder = new DistributedHerder( distributedConfig, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, advertisedUrl.toString(), connectorClientConfigOverridePolicy); // Initialize HerderStatusMonitor boolean autoStartTasks = mirusConfig.getTaskAutoRestart(); boolean autoStartConnectors = mirusConfig.getConnectorAutoRestart(); long pollingCycle = mirusConfig.getTaskStatePollingInterval(); HerderStatusMonitor herderStatusMonitor = new HerderStatusMonitor( herder, workerId, pollingCycle, autoStartTasks, autoStartConnectors); Thread herderStatusMonitorThread = new Thread(herderStatusMonitor); herderStatusMonitorThread.setName("herder-status-monitor"); final Connect connect = new Connect(herder, rest); log.info("Mirus worker initialization took {}ms", time.hiResClockMs() - initStart); try { connect.start(); } catch (Exception e) { log.error("Failed to start Mirus", e); connect.stop(); Exit.exit(3); } herderStatusMonitorThread.start(); return connect; }