org.apache.kafka.connect.runtime.WorkerConfig Java Examples
The following examples show how to use
org.apache.kafka.connect.runtime.WorkerConfig.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Compatibility.java From apicurio-registry with Apache License 2.0 | 6 votes |
static Worker createWorker(String workerId, Time time, Plugins plugins, WorkerConfig config, OffsetBackingStore offsetBackingStore, Object connectorClientConfigOverridePolicy) throws ConnectException { if (CTR_WORKER_22 == null) { return new Worker(workerId, time, plugins, config, offsetBackingStore, (ConnectorClientConfigOverridePolicy)connectorClientConfigOverridePolicy); } try { return (Worker)CTR_WORKER_22.newInstance(workerId, time, plugins, config, offsetBackingStore); } catch (Throwable t) { throw new ConnectException(t); } }
Example #2
Source File: IgniteSourceConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { kafkaBroker = new TestKafkaBroker(); Map<String, String> props = makeWorkerProps(); WorkerConfig workerCfg = new StandaloneConfig(props); MemoryOffsetBackingStore offBackingStore = new MemoryOffsetBackingStore(); offBackingStore.configure(workerCfg); worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore); worker.start(); herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg)); herder.start(); }
Example #3
Source File: IgniteSourceConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** * Creates properties for Kafka Connect workers. * * @return Worker configurations. * @throws IOException If failed. */ private Map<String, String> makeWorkerProps() throws IOException { Map<String, String> props = new HashMap<>(); props.put(WorkerConfig.INTERNAL_KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put(WorkerConfig.INTERNAL_VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put("internal.key.converter.schemas.enable", "false"); props.put("internal.value.converter.schemas.enable", "false"); props.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.ignite.stream.kafka.connect.serialization.CacheEventConverter"); props.put("key.converter.schemas.enable", "false"); props.put("value.converter.schemas.enable", "false"); props.put(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker.getBrokerAddress()); props.put("offset.storage.file.filename", "/tmp/connect.offsets"); // fast flushing for testing. props.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "10"); return props; }
Example #4
Source File: IgniteSinkConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** {@inheritDoc} */ @Override protected void beforeTest() throws Exception { kafkaBroker = new TestKafkaBroker(); for (String topic : TOPICS) kafkaBroker.createTopic(topic, PARTITIONS, REPLICATION_FACTOR); Map<String, String> props = makeWorkerProps(); WorkerConfig workerCfg = new StandaloneConfig(props); OffsetBackingStore offBackingStore = mock(OffsetBackingStore.class); offBackingStore.configure(workerCfg); worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore); worker.start(); herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg)); herder.start(); }
Example #5
Source File: IgniteSinkConnectorTest.java From ignite with Apache License 2.0 | 6 votes |
/** * Creates properties for Kafka Connect workers. * * @return Worker configurations. */ private Map<String, String> makeWorkerProps() { Map<String, String> props = new HashMap<>(); props.put(WorkerConfig.INTERNAL_KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put(WorkerConfig.INTERNAL_VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put("internal.key.converter.schemas.enable", "false"); props.put("internal.value.converter.schemas.enable", "false"); props.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.storage.StringConverter"); props.put("key.converter.schemas.enable", "false"); props.put("value.converter.schemas.enable", "false"); props.put(WorkerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBroker.getBrokerAddress()); props.put("offset.storage.file.filename", "/tmp/connect.offsets"); // fast flushing for testing. props.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, "10"); return props; }
Example #6
Source File: Compatibility.java From apicurio-registry with Apache License 2.0 | 5 votes |
static Object createConnectorClientConfigOverridePolicy(Plugins plugins, AbstractConfig config) throws ConnectException { if (CLS_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY != null) { return plugins.newPlugin( config.getString(WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG), config, CLS_CONNECTOR_CLIENT_CONFIG_OVERRIDE_POLICY); } return null; }
Example #7
Source File: EmbeddedKafkaService.java From camel-kafka-connector with Apache License 2.0 | 5 votes |
private void buildCluster() { LOG.info("Creating the embedded Kafka connect instance"); EmbeddedConnectCluster.Builder builder = new EmbeddedConnectCluster.Builder(); Properties brokerProps = new Properties(); brokerProps.put("auto.create.topics.enable", String.valueOf(true)); Map<String, String> workerProps = new HashMap<>(); workerProps.put(WorkerConfig.OFFSET_COMMIT_INTERVAL_MS_CONFIG, String.valueOf(OFFSET_COMMIT_INTERVAL_MS)); String pluginPaths = PluginPathHelper.getInstance().pluginPaths(); LOG.info("Adding the returned directories to the plugin path. This may take A VERY long time to complete"); workerProps.put(WorkerConfig.PLUGIN_PATH_CONFIG, pluginPaths); LOG.info("Building the embedded Kafka connect instance"); this.cluster = builder .name("connect-cluster") .numWorkers(1) .numBrokers(1) .brokerProps(brokerProps) .workerProps(workerProps) .maskExitProcedures(true) .build(); LOG.info("Built the embedded Kafka connect instance"); }
Example #8
Source File: OffsetFetcher.java From mirus with BSD 3-Clause "New" or "Revised" License | 5 votes |
OffsetFetcher(final WorkerConfig config, Converter internalConverter) { String topic = config.getString(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG); if ("".equals(topic)) { throw new ConfigException("Offset storage topic must be specified"); } Map<String, Object> producerProps = new HashMap<>(config.originals()); producerProps.put( ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); producerProps.put( ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, ByteArraySerializer.class.getName()); producerProps.put(ProducerConfig.RETRIES_CONFIG, Integer.MAX_VALUE); Map<String, Object> consumerProps = new HashMap<>(config.originals()); consumerProps.put( ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); consumerProps.put( ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, ByteArrayDeserializer.class.getName()); Callback<ConsumerRecord<byte[], byte[]>> consumedCallback = (error, record) -> { ByteBuffer key = record.key() != null ? ByteBuffer.wrap(record.key()) : null; ByteBuffer value = record.value() != null ? ByteBuffer.wrap(record.value()) : null; data.put(key, value); }; this.offsetLog = new KafkaBasedLog<>( topic, producerProps, consumerProps, consumedCallback, Time.SYSTEM, null); this.internalConverter = internalConverter; }
Example #9
Source File: PulsarOffsetBackingStore.java From pulsar with Apache License 2.0 | 5 votes |
@Override public void configure(WorkerConfig workerConfig) { this.topic = workerConfig.getString(PulsarKafkaWorkerConfig.OFFSET_STORAGE_TOPIC_CONFIG); checkArgument(!isBlank(topic), "Offset storage topic must be specified"); this.serviceUrl = workerConfig.getString(PulsarKafkaWorkerConfig.PULSAR_SERVICE_URL_CONFIG); checkArgument(!isBlank(serviceUrl), "Pulsar service url must be specified at `" + WorkerConfig.BOOTSTRAP_SERVERS_CONFIG + "`"); this.data = new HashMap<>(); log.info("Configure offset backing store on pulsar topic {} at cluster {}", topic, serviceUrl); }
Example #10
Source File: Mirus.java From mirus with BSD 3-Clause "New" or "Revised" License | 4 votes |
/** * This method is based on the the standard Kafka Connect start logic in {@link * org.apache.kafka.connect.cli.ConnectDistributed#startConnect(Map)}, but with `clientid` prefix * support, to prevent JMX metric names from clashing. Also supports command-line property * overrides (useful for run-time port configuration), and starts the Mirus {@link * HerderStatusMonitor}. */ public Connect startConnect(Map<String, String> workerProps) { log.info("Scanning for plugin classes. This might take a moment ..."); Plugins plugins = new Plugins(workerProps); plugins.compareAndSwapWithDelegatingLoader(); DistributedConfig distributedConfig = configWithClientIdSuffix(workerProps, "herder"); MirusConfig mirusConfig = new MirusConfig(workerProps); String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(distributedConfig); log.debug("Kafka cluster ID: {}", kafkaClusterId); RestServer rest = new RestServer(configWithClientIdSuffix(workerProps, "rest")); rest.initializeServer(); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(configWithClientIdSuffix(workerProps, "offset")); WorkerConfig workerConfigs = configWithClientIdSuffix(workerProps, "worker"); ConnectorClientConfigOverridePolicy connectorClientConfigOverridePolicy = plugins.newPlugin( distributedConfig.getString(WorkerConfig.CONNECTOR_CLIENT_POLICY_CLASS_CONFIG), workerConfigs, ConnectorClientConfigOverridePolicy.class); Worker worker = new Worker( workerId, time, plugins, workerConfigs, offsetBackingStore, connectorClientConfigOverridePolicy); WorkerConfigTransformer configTransformer = worker.configTransformer(); Converter internalValueConverter = worker.getInternalValueConverter(); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter); statusBackingStore.configure(configWithClientIdSuffix(workerProps, "status")); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore( internalValueConverter, configWithClientIdSuffix(workerProps, "config"), configTransformer); DistributedHerder herder = new DistributedHerder( distributedConfig, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, advertisedUrl.toString(), connectorClientConfigOverridePolicy); // Initialize HerderStatusMonitor boolean autoStartTasks = mirusConfig.getTaskAutoRestart(); boolean autoStartConnectors = mirusConfig.getConnectorAutoRestart(); long pollingCycle = mirusConfig.getTaskStatePollingInterval(); HerderStatusMonitor herderStatusMonitor = new HerderStatusMonitor( herder, workerId, pollingCycle, autoStartTasks, autoStartConnectors); Thread herderStatusMonitorThread = new Thread(herderStatusMonitor); herderStatusMonitorThread.setName("herder-status-monitor"); final Connect connect = new Connect(herder, rest); log.info("Mirus worker initialization took {}ms", time.hiResClockMs() - initStart); try { connect.start(); } catch (Exception e) { log.error("Failed to start Mirus", e); connect.stop(); Exit.exit(3); } herderStatusMonitorThread.start(); return connect; }