kafka.server.KafkaServerStartable Java Examples
The following examples show how to use
kafka.server.KafkaServerStartable.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaUnit.java From SkaETL with Apache License 2.0 | 6 votes |
public void startup() { zookeeper = new Zookeeper(zkPort, zkMaxConnections); zookeeper.startup(); try { logDir = Files.createTempDirectory("kafka").toFile(); } catch (IOException e) { throw new RuntimeException("Unable to start Kafka", e); } logDir.deleteOnExit(); Runtime.getRuntime().addShutdownHook(new Thread(getDeleteLogDirectoryAction())); kafkaBrokerConfig.setProperty("zookeeper.connect", zookeeperString); kafkaBrokerConfig.setProperty("broker.id", "1"); kafkaBrokerConfig.setProperty("host.name", "localhost"); kafkaBrokerConfig.setProperty("port", Integer.toString(brokerPort)); kafkaBrokerConfig.setProperty("log.dir", logDir.getAbsolutePath()); kafkaBrokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1)); kafkaBrokerConfig.setProperty("delete.topic.enable", String.valueOf(true)); kafkaBrokerConfig.setProperty("offsets.topic.replication.factor", String.valueOf(1)); kafkaBrokerConfig.setProperty("auto.create.topics.enable", String.valueOf(false)); broker = new KafkaServerStartable(new KafkaConfig(kafkaBrokerConfig)); broker.startup(); }
Example #2
Source File: KafkaOperatorTestBase.java From attic-apex-malhar with Apache License 2.0 | 6 votes |
public void startKafkaServer(int clusterid, int brokerid, int defaultPartitions) { // before start, clean the kafka dir if it exists FileUtils.deleteQuietly(new File(baseDir, kafkaBaseDir)); Properties props = new Properties(); props.setProperty("broker.id", "" + brokerid); props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid][brokerid]).toString()); props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]); props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid][brokerid]); props.setProperty("default.replication.factor", "1"); // set this to 50000 to boost the performance so most test data are in memory before flush to disk props.setProperty("log.flush.interval.messages", "50000"); if (hasMultiPartition) { props.setProperty("num.partitions", "2"); } else { props.setProperty("num.partitions", "1"); } broker[clusterid][brokerid] = new KafkaServerStartable(new KafkaConfig(props)); broker[clusterid][brokerid].startup(); }
Example #3
Source File: KafkaEmbedded.java From eagle with Apache License 2.0 | 6 votes |
public KafkaEmbedded(Integer kafkaPort, Integer zookeeperPort) { try { zk = new ZookeeperEmbedded(zookeeperPort); zk.start(); this.port = null != kafkaPort ? kafkaPort : InstanceSpec.getRandomPort(); logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/kafka-test-" + kafkaPort); FileUtils.deleteQuietly(logDir); KafkaConfig config = buildKafkaConfig(zk.getConnectionString()); kafka = new KafkaServerStartable(config); kafka.startup(); } catch (Exception ex) { throw new RuntimeException("Could not start test broker", ex); } }
Example #4
Source File: KafkaTestServerImpl.java From eagle with Apache License 2.0 | 6 votes |
@Override public void start() throws Exception { zkServer = new TestingServer(zookeeperPort, logDir); ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3); curatorClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), retryPolicy); curatorClient.start(); Properties props = new Properties(); props.setProperty("zookeeper.connect", zkServer.getConnectString()); props.setProperty("broker.id", "0"); props.setProperty("port", "" + kafkaPort); props.setProperty("log.dirs", logDir.getAbsolutePath()); props.setProperty("auto.create.topics.enable", "true"); kafkaServer = new KafkaServerStartable(new KafkaConfig(props)); kafkaServer.startup(); }
Example #5
Source File: EmbeddedKafka.java From tajo with Apache License 2.0 | 6 votes |
EmbeddedKafka(EmbeddedZookeeper zookeeper, int kafkaPort) throws IOException { this.zookeeper = checkNotNull(zookeeper, "zookeeper is null"); this.port = kafkaPort; this.kafkaDataDir = Files.createTempDir(); Properties properties = new Properties(); properties.setProperty("broker.id", "0"); properties.setProperty("host.name", "localhost"); properties.setProperty("num.partitions", "2"); properties.setProperty("log.flush.interval.messages", "10000"); properties.setProperty("log.flush.interval.ms", "1000"); properties.setProperty("log.retention.minutes", "60"); properties.setProperty("log.segment.bytes", "1048576"); properties.setProperty("auto.create.topics.enable", "false"); properties.setProperty("zookeeper.connection.timeout.ms", "1000000"); properties.setProperty("port", Integer.toString(port)); properties.setProperty("log.dirs", kafkaDataDir.getAbsolutePath()); properties.setProperty("zookeeper.connect", zookeeper.getConnectString()); KafkaConfig config = new KafkaConfig(properties); this.kafka = new KafkaServerStartable(config); }
Example #6
Source File: KafkaTestCase.java From product-cep with Apache License 2.0 | 6 votes |
private void setupKafkaBroker() { try { // mock zookeeper zkTestServer = new TestingServer(2181); // mock kafka Properties props = new Properties(); props.put("broker.id", "0"); props.put("host.name", "localhost"); props.put("port", "9092"); props.put("log.dir", "/tmp/tmp_kafka_dir"); props.put("zookeeper.connect", zkTestServer.getConnectString()); props.put("replica.socket.timeout.ms", "1500"); KafkaConfig config = new KafkaConfig(props); kafkaServer = new KafkaServerStartable(config); kafkaServer.startup(); // create "sensordata" topic ZkClient zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$); AdminUtils.createTopic(zkClient, "sensordata", 1, 1, new Properties()); zkClient.close(); } catch (Exception e) { log.error("Error running local Kafka broker / Zookeeper", e); } }
Example #7
Source File: EmbeddedKafkaCluster.java From beam with Apache License 2.0 | 6 votes |
public void startup() { for (int i = 0; i < ports.size(); i++) { Integer port = ports.get(i); File logDir = TestUtils.constructTempDir("kafka-local"); Properties properties = new Properties(); properties.putAll(baseProperties); properties.setProperty("zookeeper.connect", zkConnection); properties.setProperty("broker.id", String.valueOf(i + 1)); properties.setProperty("advertised.host.name", "127.0.0.1"); properties.setProperty("host.name", "127.0.0.1"); properties.setProperty("advertised.port", Integer.toString(port)); properties.setProperty("port", Integer.toString(port)); properties.setProperty("log.dirs", logDir.getAbsolutePath()); properties.setProperty("offsets.topic.num.partitions", "1"); properties.setProperty("offsets.topic.replication.factor", "1"); properties.setProperty("log.flush.interval.messages", String.valueOf(1)); KafkaServerStartable broker = startBroker(properties); brokers.add(broker); logDirs.add(logDir); } }
Example #8
Source File: LocalKafkaBroker.java From oryx with Apache License 2.0 | 6 votes |
/** * Starts the Kafka broker. * * @throws IOException if an error occurs during initialization */ public synchronized void start() throws IOException { log.info("Starting Kafka broker on port {}", port); logsDir = Files.createTempDirectory(LocalKafkaBroker.class.getSimpleName()); logsDir.toFile().deleteOnExit(); kafkaServer = new KafkaServerStartable(new KafkaConfig(ConfigUtils.keyValueToProperties( "broker.id", TEST_BROKER_ID, "log.dirs", logsDir.toAbsolutePath(), "listeners", "PLAINTEXT://:" + port, "zookeeper.connect", "localhost:" + zkPort, "message.max.bytes", 1 << 26, "replica.fetch.max.bytes", 1 << 26, "offsets.topic.replication.factor", 1 ), false)); kafkaServer.startup(); }
Example #9
Source File: LocalKafkaServer.java From Krackle with Apache License 2.0 | 6 votes |
public LocalKafkaServer() throws IOException { while (new File(logDir).exists()) { FileUtils.deleteDirectory(new File(logDir)); } Properties props = new Properties(); props.put("broker.id", nodeId); props.put("port", port); props.put("log.dir", logDir); props.put("zookeeper.connect", zkConnect); props.put("host.name", "127.0.0.1"); KafkaConfig conf = new KafkaConfig(props); zkUtils = ZkUtils.apply(props.getProperty("zookeeper.connect"), 30000, 30000, JaasUtils.isZkSecurityEnabled()); server = new KafkaServerStartable(conf); server.startup(); }
Example #10
Source File: KafkaStarterUtils.java From uReplicator with Apache License 2.0 | 6 votes |
public static KafkaServerStartable startServer(final int port, final int brokerId, final String zkStr, final Properties configuration) { // Create the ZK nodes for Kafka, if needed int indexOfFirstSlash = zkStr.indexOf('/'); if (indexOfFirstSlash != -1) { String bareZkUrl = zkStr.substring(0, indexOfFirstSlash); String zkNodePath = zkStr.substring(indexOfFirstSlash); ZkClient client = new ZkClient(bareZkUrl); client.createPersistent(zkNodePath, true); client.close(); } File logDir = new File("/tmp/kafka-" + Double.toHexString(Math.random())); logDir.mkdirs(); logDir.deleteOnExit(); kafkaDataDir = logDir.toString(); configureKafkaPort(configuration, port); configureZkConnectionString(configuration, zkStr); configureBrokerId(configuration, brokerId); configureKafkaLogDirectory(configuration, logDir); KafkaConfig config = new KafkaConfig(configuration); KafkaServerStartable serverStartable = new KafkaServerStartable(config); serverStartable.startup(); return serverStartable; }
Example #11
Source File: KafkaDataServerStartable.java From incubator-pinot with Apache License 2.0 | 6 votes |
public void init(Properties props) { port = (int) props.get(PORT); zkStr = props.getProperty(ZOOKEEPER_CONNECT); logDirPath = props.getProperty(LOG_DIRS); // Create the ZK nodes for Kafka, if needed int indexOfFirstSlash = zkStr.indexOf('/'); if (indexOfFirstSlash != -1) { String bareZkUrl = zkStr.substring(0, indexOfFirstSlash); String zkNodePath = zkStr.substring(indexOfFirstSlash); ZkClient client = new ZkClient(bareZkUrl); client.createPersistent(zkNodePath, true); client.close(); } File logDir = new File(logDirPath); logDir.mkdirs(); props.put("zookeeper.session.timeout.ms", "60000"); serverStartable = new KafkaServerStartable(new KafkaConfig(props)); final Map<String, Object> config = new HashMap<>(); config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port); config.put(AdminClientConfig.CLIENT_ID_CONFIG, "Kafka2AdminClient-" + UUID.randomUUID().toString()); config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, 15000); adminClient = KafkaAdminClient.create(config); }
Example #12
Source File: KafkaTestUtil.java From siddhi-io-kafka with Apache License 2.0 | 6 votes |
public static void setupKafkaBroker2() { try { log.info("#############################################################################################"); log.info("################################# ZOOKEEPER 2 STARTED ####################################"); log.info("#############################################################################################"); // mock zookeeper zkTestServer2 = new TestingServer(2182); // mock kafka Properties props = new Properties(); props.put("broker.id", "1"); props.put("host.name", "localhost"); props.put("port", "9093"); props.put("log.dir", kafkaLogDir2); props.put("zookeeper.connect", zkTestServer2.getConnectString()); props.put("replica.socket.timeout.ms", "30000"); props.put("delete.topic.enable", "true"); props.put("log.cleaner.dedupe.buffer.size", CLEANER_BUFFER_SIZE); KafkaConfig config = new KafkaConfig(props); kafkaServer2 = new KafkaServerStartable(config); kafkaServer2.startup(); } catch (Exception e) { log.error("Error running local Kafka broker 2", e); } }
Example #13
Source File: KafkaTestUtil.java From siddhi-io-kafka with Apache License 2.0 | 6 votes |
public static void setupKafkaBroker() { try { log.info("#############################################################################################"); log.info("################################# ZOOKEEPER STARTED ######################################"); log.info("#############################################################################################"); // mock zookeeper zkTestServer = new TestingServer(2181); // mock kafka Properties props = new Properties(); props.put("broker.id", "0"); props.put("host.name", "localhost"); props.put("port", "9092"); props.put("log.dir", kafkaLogDir); props.put("zookeeper.connect", zkTestServer.getConnectString()); props.put("replica.socket.timeout.ms", "30000"); props.put("delete.topic.enable", "true"); props.put("log.cleaner.dedupe.buffer.size", CLEANER_BUFFER_SIZE); KafkaConfig config = new KafkaConfig(props); kafkaServer = new KafkaServerStartable(config); kafkaServer.startup(); } catch (Exception e) { log.error("Error running local Kafka broker / Zookeeper", e); } }
Example #14
Source File: KafkaDataServerStartable.java From incubator-pinot with Apache License 2.0 | 6 votes |
public void init(Properties props) { zkStr = props.getProperty(ZOOKEEPER_CONNECT); logDirPath = props.getProperty(LOG_DIRS); // Create the ZK nodes for Kafka, if needed int indexOfFirstSlash = zkStr.indexOf('/'); if (indexOfFirstSlash != -1) { String bareZkUrl = zkStr.substring(0, indexOfFirstSlash); String zkNodePath = zkStr.substring(indexOfFirstSlash); ZkClient client = new ZkClient(bareZkUrl); client.createPersistent(zkNodePath, true); client.close(); } File logDir = new File(logDirPath); logDir.mkdirs(); props.put("zookeeper.session.timeout.ms", "60000"); KafkaConfig config = new KafkaConfig(props); serverStartable = new KafkaServerStartable(config); }
Example #15
Source File: TestUtil.java From feast with Apache License 2.0 | 6 votes |
/** * Start local Kafka and (optionally) Zookeeper * * @param kafkaHost e.g. localhost * @param kafkaPort e.g. 60001 * @param kafkaReplicationFactor e.g. 1 * @param zookeeperHost e.g. localhost * @param zookeeperPort e.g. 60002 * @param zookeeperDataDir e.g. "/tmp" or "Files.createTempDir().getAbsolutePath()" */ public static void start( String kafkaHost, int kafkaPort, short kafkaReplicationFactor, boolean startZookeper, String zookeeperHost, int zookeeperPort, String zookeeperDataDir) throws InterruptedException { if (startZookeper) { LocalZookeeper.start(zookeeperPort, zookeeperDataDir); Thread.sleep(5000); } Properties kafkaProp = new Properties(); kafkaProp.put("zookeeper.connect", zookeeperHost + ":" + zookeeperPort); kafkaProp.put("host.name", kafkaHost); kafkaProp.put("port", kafkaPort); kafkaProp.put("log.dir", Files.createTempDir().getAbsolutePath()); kafkaProp.put("offsets.topic.replication.factor", kafkaReplicationFactor); KafkaConfig kafkaConfig = new KafkaConfig(kafkaProp); server = new KafkaServerStartable(kafkaConfig); new Thread(server::startup).start(); Thread.sleep(5000); }
Example #16
Source File: KafkaServer.java From ingestion with Apache License 2.0 | 5 votes |
public void start() throws Exception { File dir = Files.createTempDir(); String dirPath = dir.getAbsolutePath(); System.out.println("Storing Kafka files in " + dirPath); String localhostKafka = System.getProperty("kafka.ip"); String localhostZookeeper = System.getProperty("zookeeper.ip"); String portKafka = System.getProperty("kafka.port"); String portZookeeper = System.getProperty("zookeeper.port"); Properties properties = new Properties(); properties.put("broker.id", "0"); properties.put("host.name", "localhost"); // properties.put("host.name", localhostKafka); properties.put("port", KAFKA_PORT); properties.put("log.dir", dirPath); properties.put("log.flush.interval.messages", "1"); // properties.put("zookeeper.connect", "localhost:" + ZookeeperServer.CLIENT_PORT); properties.put("zookeeper.connect", localhostZookeeper+":" + ZookeeperServer.CLIENT_PORT); // properties.put("zookeeper.connect", "port:" + ZookeeperServer.CLIENT_PORT); // Integer clientPort = Integer.parseInt(System.getProperty("zookeeper.port")); // properties.put("zookeeper.connect", "localhost:" + clientPort); properties.put("replica.socket.timeout.ms", "1500"); properties.put("auto.create.topics.enable", "true"); properties.put("num.partitions", "1"); KafkaConfig kafkaConfig = new KafkaConfig(properties); kafkaServer = new KafkaServerStartable(kafkaConfig); kafkaServer.startup(); TimeUnit.SECONDS.sleep(WAIT_SECONDS); }
Example #17
Source File: EmbeddedKafka.java From nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.kafkaConfig = kafkaConfig; this.zookeeperConfig = zookeeperConfig; this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); this.zkServer = new ZooKeeperServer(); }
Example #18
Source File: EmbeddedKafka.java From nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.zookeeperConfig = zookeeperConfig; this.kafkaConfig = kafkaConfig; this.kafkaPort = this.availablePort(); this.zookeeperPort = this.availablePort(); this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort)); this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort); this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort)); this.zkServer = new ZooKeeperServer(); this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); }
Example #19
Source File: EmbeddedKafka.java From nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.zookeeperConfig = zookeeperConfig; this.kafkaConfig = kafkaConfig; this.kafkaPort = this.availablePort(); this.zookeeperPort = this.availablePort(); this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort)); this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort); this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort)); this.zkServer = new ZooKeeperServer(); this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); }
Example #20
Source File: EmbeddedKafka.java From nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.zookeeperConfig = zookeeperConfig; this.kafkaConfig = kafkaConfig; this.kafkaPort = this.availablePort(); this.zookeeperPort = this.availablePort(); this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort)); this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort); this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort)); this.zkServer = new ZooKeeperServer(); this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); }
Example #21
Source File: KafkaOperatorTestBase.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public static void startKafkaServer(int clusterid, int brokerid) { Properties props = new Properties(); props.setProperty("broker.id", "" + clusterid * 10 + brokerid); props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString()); props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]); props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]); props.setProperty("default.replication.factor", "1"); // set this to 50000 to boost the performance so most test data are in memory before flush to disk props.setProperty("log.flush.interval.messages", "50000"); broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props)); broker[clusterid].startup(); }
Example #22
Source File: KafkaOperatorTestBase.java From attic-apex-malhar with Apache License 2.0 | 5 votes |
public static void startKafkaServer(int clusterid, int brokerid) { Properties props = new Properties(); props.setProperty("broker.id", "" + clusterid * 10 + brokerid); props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString()); props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]); props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]); props.setProperty("default.replication.factor", "1"); // set this to 50000 to boost the performance so most test data are in memory before flush to disk props.setProperty("log.flush.interval.messages", "50000"); broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props)); broker[clusterid].startup(); }
Example #23
Source File: EmbeddedKafka.java From localization_nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.zookeeperConfig = zookeeperConfig; this.kafkaConfig = kafkaConfig; this.kafkaPort = this.availablePort(); this.zookeeperPort = this.availablePort(); this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort)); this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort); this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort)); this.zkServer = new ZooKeeperServer(); this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); }
Example #24
Source File: EmbeddedKafka.java From localization_nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.zookeeperConfig = zookeeperConfig; this.kafkaConfig = kafkaConfig; this.kafkaPort = this.availablePort(); this.zookeeperPort = this.availablePort(); this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort)); this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort); this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort)); this.zkServer = new ZooKeeperServer(); this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); }
Example #25
Source File: EmbeddedKafka.java From localization_nifi with Apache License 2.0 | 5 votes |
/** * Will create instance of the embedded Kafka server. * * @param kafkaConfig * Kafka configuration properties * @param zookeeperConfig * Zookeeper configuration properties */ public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) { this.cleanupKafkaWorkDir(); this.zookeeperConfig = zookeeperConfig; this.kafkaConfig = kafkaConfig; this.kafkaPort = this.availablePort(); this.zookeeperPort = this.availablePort(); this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort)); this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort); this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort)); this.zkServer = new ZooKeeperServer(); this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig)); }
Example #26
Source File: KafkaLocal.java From sqoop-on-spark with Apache License 2.0 | 5 votes |
public KafkaLocal(Properties kafkaProperties) throws IOException, InterruptedException{ kafkaConfig = new KafkaConfig(kafkaProperties); //start local kafka broker kafka = new KafkaServerStartable(kafkaConfig); }
Example #27
Source File: EmbeddedKafka.java From mercury with Apache License 2.0 | 5 votes |
@Override public void run() { try (InputStream stream = EmbeddedKafka.class.getResourceAsStream("/kafka.properties")) { if (stream == null) { throw new IOException("kafka.properties is not available as resource"); } Properties p = new Properties(); p.load(stream); String dir = p.getProperty("log.dirs"); if (dir != null) { File reset = new File(dir); if (reset.exists() && reset.isDirectory()) { Utility.getInstance().cleanupDir(reset); log.info("Clean up transient Kafka working directory at {}", dir); } } kafka = new KafkaServerStartable(new KafkaConfig(p)); kafka.startup(); Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown)); } catch (IOException e) { log.error("Unable to start Kafka kafka - {}", e.getMessage()); System.exit(-1); } }
Example #28
Source File: LocalKafka.java From beam with Apache License 2.0 | 5 votes |
LocalKafka(int kafkaPort, int zookeeperPort) throws Exception { Properties kafkaProperties = new Properties(); kafkaProperties.setProperty("port", String.valueOf(kafkaPort)); kafkaProperties.setProperty("zookeeper.connect", String.format("localhost:%s", zookeeperPort)); kafkaProperties.setProperty("offsets.topic.replication.factor", "1"); kafkaProperties.setProperty("log.dir", Files.createTempDirectory("kafka-log-").toString()); server = new KafkaServerStartable(KafkaConfig.fromProps(kafkaProperties)); }
Example #29
Source File: KafkaServer.java From hbase-connect-kafka with Apache License 2.0 | 5 votes |
public KafkaServer(int zookeeperPort, int kafkaBrokerPort) { try { Preconditions.checkArgument(zookeeperPort > 0); Preconditions.checkArgument(kafkaBrokerPort > 0); this.zookeeperPort = zookeeperPort; this.brokerPort = kafkaBrokerPort; this.logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/hbase-cdc-kafka-" + brokerPort); KafkaConfig config = buildKafkaConfig(zookeeperPort); kafka = new KafkaServerStartable(config); kafka.startup(); } catch (Exception ex) { throw new RuntimeException("Could not start test broker", ex); } }
Example #30
Source File: KafkaStarterUtils.java From uReplicator with Apache License 2.0 | 5 votes |
public static void stopServer(KafkaServerStartable serverStartable) { serverStartable.shutdown(); if (StringUtils.isNotEmpty(kafkaDataDir)) { try { FileUtils.deleteDirectory(new File(kafkaDataDir)); } catch (IOException e) { LOGGER.warn("Caught exception while stopping kafka", e); } } }