org.apache.kafka.connect.runtime.standalone.StandaloneHerder Java Examples
The following examples show how to use
org.apache.kafka.connect.runtime.standalone.StandaloneHerder.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IgniteSourceConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { kafkaBroker = new TestKafkaBroker(); Map<String, String> props = makeWorkerProps(); WorkerConfig workerCfg = new StandaloneConfig(props); MemoryOffsetBackingStore offBackingStore = new MemoryOffsetBackingStore(); offBackingStore.configure(workerCfg); worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore); worker.start(); herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg)); herder.start(); }
Example #2
Source File: IgniteSinkConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { kafkaBroker = new TestKafkaBroker(); for (String topic : TOPICS) kafkaBroker.createTopic(topic, PARTITIONS, REPLICATION_FACTOR); Map<String, String> props = makeWorkerProps(); WorkerConfig workerCfg = new StandaloneConfig(props); OffsetBackingStore offBackingStore = mock(OffsetBackingStore.class); offBackingStore.configure(workerCfg); worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore); worker.start(); herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg)); herder.start(); }
Example #3
Source File: KafkaConnectRunner.java From camel-kafka-connector with Apache License 2.0 | 5 votes |
/** * here does not seem to be a public interface for embedding a Kafka connect runtime, * therefore, this code is modeled from the behavior taken from * https://github.com/apache/kafka/blob/2.1/connect/runtime/src/main/java/org/apache/kafka/connect/cli/ConnectStandalone.java * and performs the initialization in a roughly similar manner. * */ private void init() { LOG.info("Started worked initialization"); Time time = Time.SYSTEM; // Initializes the system runtime information and logs some of the information WorkerInfo initInfo = new WorkerInfo(); initInfo.logAll(); Properties props = kafkaConnectPropertyFactory.getProperties(); Map<String, String> standAloneProperties = Utils.propsToStringMap(props); // Not needed, but we need this one to initialize the worker Plugins plugins = new Plugins(standAloneProperties); StandaloneConfig config = new StandaloneConfig(standAloneProperties); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config); AllConnectorClientConfigOverridePolicy allConnectorClientConfigOverridePolicy = new AllConnectorClientConfigOverridePolicy(); RestServer rest = new RestServer(config); rest.initializeServer(); /* According to the Kafka source code "... Worker runs a (dynamic) set of tasks in a set of threads, doing the work of actually moving data to/from Kafka ..." */ Worker worker = new Worker(bootstrapServer, time, plugins, config, new FileOffsetBackingStore(), allConnectorClientConfigOverridePolicy); /* From Kafka source code: " ... The herder interface tracks and manages workers and connectors ..." */ herder = new StandaloneHerder(worker, kafkaClusterId, allConnectorClientConfigOverridePolicy); connect = new Connect(herder, rest); LOG.info("Finished initializing the worker"); }
Example #4
Source File: ConnectRunner.java From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0 | 5 votes |
void start() { final Map<String, String> workerProps = new HashMap<>(); workerProps.put("bootstrap.servers", bootstrapServers); workerProps.put("offset.flush.interval.ms", Integer.toString(offsetFlushInterval)); // These don't matter much (each connector sets its own converters), but need to be filled with valid classes. workerProps.put("key.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); workerProps.put("value.converter", "org.apache.kafka.connect.converters.ByteArrayConverter"); workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter"); workerProps.put("internal.key.converter.schemas.enable", "false"); workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter"); workerProps.put("internal.value.converter.schemas.enable", "false"); // Don't need it since we'll memory MemoryOffsetBackingStore. workerProps.put("offset.storage.file.filename", ""); workerProps.put("plugin.path", pluginDir.getPath()); final Time time = Time.SYSTEM; final String workerId = "test-worker"; final Plugins plugins = new Plugins(workerProps); final StandaloneConfig config = new StandaloneConfig(workerProps); final Worker worker = new Worker( workerId, time, plugins, config, new MemoryOffsetBackingStore()); herder = new StandaloneHerder(worker); final RestServer rest = new RestServer(config); connect = new Connect(herder, rest); connect.start(); }
Example #5
Source File: ConnectStandalone.java From mongo-kafka with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") ConnectStandalone(final Properties workerProperties) { Time time = Time.SYSTEM; LOGGER.info("Kafka Connect standalone worker initializing ..."); long initStart = time.hiResClockMs(); WorkerInfo initInfo = new WorkerInfo(); initInfo.logAll(); Map<String, String> workerProps = (Map) workerProperties; LOGGER.info("Scanning for plugin classes. This might take a moment ..."); Plugins plugins = new Plugins(workerProps); plugins.compareAndSwapWithDelegatingLoader(); StandaloneConfig config = new StandaloneConfig(workerProps); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config); LOGGER.debug("Kafka cluster ID: {}", kafkaClusterId); RestServer rest = new RestServer(config); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore()); this.herder = new StandaloneHerder(worker, kafkaClusterId); connectionString = advertisedUrl.toString() + herder.kafkaClusterId(); this.connect = new Connect(herder, rest); LOGGER.info( "Kafka Connect standalone worker initialization took {}ms", time.hiResClockMs() - initStart); }