kafka.server.KafkaServerStartable Java Examples

The following examples show how to use kafka.server.KafkaServerStartable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaUnit.java    From SkaETL with Apache License 2.0 6 votes vote down vote up
public void startup() {
    zookeeper = new Zookeeper(zkPort, zkMaxConnections);
    zookeeper.startup();

    try {
        logDir = Files.createTempDirectory("kafka").toFile();
    } catch (IOException e) {
        throw new RuntimeException("Unable to start Kafka", e);
    }
    logDir.deleteOnExit();
    Runtime.getRuntime().addShutdownHook(new Thread(getDeleteLogDirectoryAction()));
    kafkaBrokerConfig.setProperty("zookeeper.connect", zookeeperString);
    kafkaBrokerConfig.setProperty("broker.id", "1");
    kafkaBrokerConfig.setProperty("host.name", "localhost");
    kafkaBrokerConfig.setProperty("port", Integer.toString(brokerPort));
    kafkaBrokerConfig.setProperty("log.dir", logDir.getAbsolutePath());
    kafkaBrokerConfig.setProperty("log.flush.interval.messages", String.valueOf(1));
    kafkaBrokerConfig.setProperty("delete.topic.enable", String.valueOf(true));
    kafkaBrokerConfig.setProperty("offsets.topic.replication.factor", String.valueOf(1));
    kafkaBrokerConfig.setProperty("auto.create.topics.enable", String.valueOf(false));

    broker = new KafkaServerStartable(new KafkaConfig(kafkaBrokerConfig));
    broker.startup();
}
 
Example #2
Source File: KafkaOperatorTestBase.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public void startKafkaServer(int clusterid, int brokerid, int defaultPartitions)
{
  // before start, clean the kafka dir if it exists
  FileUtils.deleteQuietly(new File(baseDir, kafkaBaseDir));

  Properties props = new Properties();
  props.setProperty("broker.id", "" + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid][brokerid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid][brokerid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");
  if (hasMultiPartition) {
    props.setProperty("num.partitions", "2");
  } else {
    props.setProperty("num.partitions", "1");
  }

  broker[clusterid][brokerid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid][brokerid].startup();

}
 
Example #3
Source File: KafkaEmbedded.java    From eagle with Apache License 2.0 6 votes vote down vote up
public KafkaEmbedded(Integer kafkaPort, Integer zookeeperPort) {
    try {
        zk = new ZookeeperEmbedded(zookeeperPort);
        zk.start();

        this.port = null != kafkaPort ? kafkaPort : InstanceSpec.getRandomPort();
        logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/kafka-test-" + kafkaPort);
        FileUtils.deleteQuietly(logDir);

        KafkaConfig config = buildKafkaConfig(zk.getConnectionString());
        kafka = new KafkaServerStartable(config);
        kafka.startup();
    } catch (Exception ex) {
        throw new RuntimeException("Could not start test broker", ex);
    }
}
 
Example #4
Source File: KafkaTestServerImpl.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public void start() throws Exception {
    zkServer = new TestingServer(zookeeperPort, logDir);
    ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
    curatorClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), retryPolicy);
    curatorClient.start();

    Properties props = new Properties();

    props.setProperty("zookeeper.connect", zkServer.getConnectString());
    props.setProperty("broker.id", "0");
    props.setProperty("port", "" + kafkaPort);
    props.setProperty("log.dirs", logDir.getAbsolutePath());
    props.setProperty("auto.create.topics.enable", "true");

    kafkaServer = new KafkaServerStartable(new KafkaConfig(props));
    kafkaServer.startup();
}
 
Example #5
Source File: EmbeddedKafka.java    From tajo with Apache License 2.0 6 votes vote down vote up
EmbeddedKafka(EmbeddedZookeeper zookeeper, int kafkaPort) throws IOException {
  this.zookeeper = checkNotNull(zookeeper, "zookeeper is null");

  this.port = kafkaPort;
  this.kafkaDataDir = Files.createTempDir();

  Properties properties = new Properties();
  properties.setProperty("broker.id", "0");
  properties.setProperty("host.name", "localhost");
  properties.setProperty("num.partitions", "2");
  properties.setProperty("log.flush.interval.messages", "10000");
  properties.setProperty("log.flush.interval.ms", "1000");
  properties.setProperty("log.retention.minutes", "60");
  properties.setProperty("log.segment.bytes", "1048576");
  properties.setProperty("auto.create.topics.enable", "false");
  properties.setProperty("zookeeper.connection.timeout.ms", "1000000");
  properties.setProperty("port", Integer.toString(port));
  properties.setProperty("log.dirs", kafkaDataDir.getAbsolutePath());
  properties.setProperty("zookeeper.connect", zookeeper.getConnectString());

  KafkaConfig config = new KafkaConfig(properties);
  this.kafka = new KafkaServerStartable(config);
}
 
Example #6
Source File: KafkaTestCase.java    From product-cep with Apache License 2.0 6 votes vote down vote up
private void setupKafkaBroker() {
    try {
        // mock zookeeper
        zkTestServer = new TestingServer(2181);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "0");
        props.put("host.name", "localhost");
        props.put("port", "9092");
        props.put("log.dir", "/tmp/tmp_kafka_dir");
        props.put("zookeeper.connect", zkTestServer.getConnectString());
        props.put("replica.socket.timeout.ms", "1500");
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer = new KafkaServerStartable(config);
        kafkaServer.startup();

        // create "sensordata" topic
        ZkClient zkClient = new ZkClient(zkTestServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);
        AdminUtils.createTopic(zkClient, "sensordata", 1, 1, new Properties());
        zkClient.close();
    } catch (Exception e) {
        log.error("Error running local Kafka broker / Zookeeper", e);
    }
}
 
Example #7
Source File: EmbeddedKafkaCluster.java    From beam with Apache License 2.0 6 votes vote down vote up
public void startup() {
  for (int i = 0; i < ports.size(); i++) {
    Integer port = ports.get(i);
    File logDir = TestUtils.constructTempDir("kafka-local");

    Properties properties = new Properties();
    properties.putAll(baseProperties);
    properties.setProperty("zookeeper.connect", zkConnection);
    properties.setProperty("broker.id", String.valueOf(i + 1));
    properties.setProperty("advertised.host.name", "127.0.0.1");
    properties.setProperty("host.name", "127.0.0.1");
    properties.setProperty("advertised.port", Integer.toString(port));
    properties.setProperty("port", Integer.toString(port));
    properties.setProperty("log.dirs", logDir.getAbsolutePath());
    properties.setProperty("offsets.topic.num.partitions", "1");
    properties.setProperty("offsets.topic.replication.factor", "1");
    properties.setProperty("log.flush.interval.messages", String.valueOf(1));

    KafkaServerStartable broker = startBroker(properties);

    brokers.add(broker);
    logDirs.add(logDir);
  }
}
 
Example #8
Source File: LocalKafkaBroker.java    From oryx with Apache License 2.0 6 votes vote down vote up
/**
 * Starts the Kafka broker.
 *
 * @throws IOException if an error occurs during initialization
 */
public synchronized void start() throws IOException {
  log.info("Starting Kafka broker on port {}", port);

  logsDir = Files.createTempDirectory(LocalKafkaBroker.class.getSimpleName());
  logsDir.toFile().deleteOnExit();

  kafkaServer = new KafkaServerStartable(new KafkaConfig(ConfigUtils.keyValueToProperties(
      "broker.id", TEST_BROKER_ID,
      "log.dirs", logsDir.toAbsolutePath(),
      "listeners", "PLAINTEXT://:" + port,
      "zookeeper.connect", "localhost:" + zkPort,
      "message.max.bytes", 1 << 26,
      "replica.fetch.max.bytes", 1 << 26,
      "offsets.topic.replication.factor", 1
  ), false));
  kafkaServer.startup();
}
 
Example #9
Source File: LocalKafkaServer.java    From Krackle with Apache License 2.0 6 votes vote down vote up
public LocalKafkaServer() throws IOException {

		while (new File(logDir).exists()) {
			FileUtils.deleteDirectory(new File(logDir));
		}

		Properties props = new Properties();
		props.put("broker.id", nodeId);
		props.put("port", port);
		props.put("log.dir", logDir);
		props.put("zookeeper.connect", zkConnect);
		props.put("host.name", "127.0.0.1");
		KafkaConfig conf = new KafkaConfig(props);

                zkUtils = ZkUtils.apply(props.getProperty("zookeeper.connect"),
                          30000,
                          30000,
                          JaasUtils.isZkSecurityEnabled());


		server = new KafkaServerStartable(conf);
		server.startup();
	}
 
Example #10
Source File: KafkaStarterUtils.java    From uReplicator with Apache License 2.0 6 votes vote down vote up
public static KafkaServerStartable startServer(final int port, final int brokerId,
    final String zkStr, final Properties configuration) {
  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File("/tmp/kafka-" + Double.toHexString(Math.random()));
  logDir.mkdirs();
  logDir.deleteOnExit();
  kafkaDataDir = logDir.toString();

  configureKafkaPort(configuration, port);
  configureZkConnectionString(configuration, zkStr);
  configureBrokerId(configuration, brokerId);
  configureKafkaLogDirectory(configuration, logDir);
  KafkaConfig config = new KafkaConfig(configuration);
  KafkaServerStartable serverStartable = new KafkaServerStartable(config);
  serverStartable.startup();
  return serverStartable;
}
 
Example #11
Source File: KafkaDataServerStartable.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
public void init(Properties props) {
  port = (int) props.get(PORT);
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  serverStartable = new KafkaServerStartable(new KafkaConfig(props));
  final Map<String, Object> config = new HashMap<>();
  config.put(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:" + port);
  config.put(AdminClientConfig.CLIENT_ID_CONFIG, "Kafka2AdminClient-" + UUID.randomUUID().toString());
  config.put(AdminClientConfig.REQUEST_TIMEOUT_MS_CONFIG, 15000);
  adminClient = KafkaAdminClient.create(config);
}
 
Example #12
Source File: KafkaTestUtil.java    From siddhi-io-kafka with Apache License 2.0 6 votes vote down vote up
public static void setupKafkaBroker2() {
    try {
        log.info("#############################################################################################");
        log.info("#################################   ZOOKEEPER 2 STARTED  ####################################");
        log.info("#############################################################################################");
        // mock zookeeper
        zkTestServer2 = new TestingServer(2182);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "1");
        props.put("host.name", "localhost");
        props.put("port", "9093");
        props.put("log.dir", kafkaLogDir2);
        props.put("zookeeper.connect", zkTestServer2.getConnectString());
        props.put("replica.socket.timeout.ms", "30000");
        props.put("delete.topic.enable", "true");
        props.put("log.cleaner.dedupe.buffer.size", CLEANER_BUFFER_SIZE);
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer2 = new KafkaServerStartable(config);
        kafkaServer2.startup();

    } catch (Exception e) {
        log.error("Error running local Kafka broker 2", e);
    }
}
 
Example #13
Source File: KafkaTestUtil.java    From siddhi-io-kafka with Apache License 2.0 6 votes vote down vote up
public static void setupKafkaBroker() {
    try {
        log.info("#############################################################################################");
        log.info("#################################   ZOOKEEPER STARTED  ######################################");
        log.info("#############################################################################################");
        // mock zookeeper
        zkTestServer = new TestingServer(2181);
        // mock kafka
        Properties props = new Properties();
        props.put("broker.id", "0");
        props.put("host.name", "localhost");
        props.put("port", "9092");
        props.put("log.dir", kafkaLogDir);
        props.put("zookeeper.connect", zkTestServer.getConnectString());
        props.put("replica.socket.timeout.ms", "30000");
        props.put("delete.topic.enable", "true");
        props.put("log.cleaner.dedupe.buffer.size", CLEANER_BUFFER_SIZE);
        KafkaConfig config = new KafkaConfig(props);
        kafkaServer = new KafkaServerStartable(config);
        kafkaServer.startup();
    } catch (Exception e) {
        log.error("Error running local Kafka broker / Zookeeper", e);
    }
}
 
Example #14
Source File: KafkaDataServerStartable.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
public void init(Properties props) {
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  KafkaConfig config = new KafkaConfig(props);

  serverStartable = new KafkaServerStartable(config);
}
 
Example #15
Source File: TestUtil.java    From feast with Apache License 2.0 6 votes vote down vote up
/**
 * Start local Kafka and (optionally) Zookeeper
 *
 * @param kafkaHost e.g. localhost
 * @param kafkaPort e.g. 60001
 * @param kafkaReplicationFactor e.g. 1
 * @param zookeeperHost e.g. localhost
 * @param zookeeperPort e.g. 60002
 * @param zookeeperDataDir e.g. "/tmp" or "Files.createTempDir().getAbsolutePath()"
 */
public static void start(
    String kafkaHost,
    int kafkaPort,
    short kafkaReplicationFactor,
    boolean startZookeper,
    String zookeeperHost,
    int zookeeperPort,
    String zookeeperDataDir)
    throws InterruptedException {
  if (startZookeper) {
    LocalZookeeper.start(zookeeperPort, zookeeperDataDir);
    Thread.sleep(5000);
  }
  Properties kafkaProp = new Properties();
  kafkaProp.put("zookeeper.connect", zookeeperHost + ":" + zookeeperPort);
  kafkaProp.put("host.name", kafkaHost);
  kafkaProp.put("port", kafkaPort);
  kafkaProp.put("log.dir", Files.createTempDir().getAbsolutePath());
  kafkaProp.put("offsets.topic.replication.factor", kafkaReplicationFactor);
  KafkaConfig kafkaConfig = new KafkaConfig(kafkaProp);
  server = new KafkaServerStartable(kafkaConfig);
  new Thread(server::startup).start();
  Thread.sleep(5000);
}
 
Example #16
Source File: KafkaServer.java    From ingestion with Apache License 2.0 5 votes vote down vote up
public void start() throws Exception {

        File dir = Files.createTempDir();
        String dirPath = dir.getAbsolutePath();
        System.out.println("Storing Kafka files in " + dirPath);

        String localhostKafka = System.getProperty("kafka.ip");
        String localhostZookeeper = System.getProperty("zookeeper.ip");
        String portKafka = System.getProperty("kafka.port");
        String portZookeeper = System.getProperty("zookeeper.port");

        Properties properties = new Properties();
        properties.put("broker.id", "0");
        properties.put("host.name", "localhost");
//        properties.put("host.name", localhostKafka);
        properties.put("port", KAFKA_PORT);
        properties.put("log.dir", dirPath);
        properties.put("log.flush.interval.messages", "1");
//        properties.put("zookeeper.connect", "localhost:" + ZookeeperServer.CLIENT_PORT);
        properties.put("zookeeper.connect", localhostZookeeper+":" + ZookeeperServer.CLIENT_PORT);
//        properties.put("zookeeper.connect", "port:" + ZookeeperServer.CLIENT_PORT);
//        Integer clientPort = Integer.parseInt(System.getProperty("zookeeper.port"));
//        properties.put("zookeeper.connect", "localhost:" + clientPort);
        properties.put("replica.socket.timeout.ms", "1500");
        properties.put("auto.create.topics.enable", "true");
        properties.put("num.partitions", "1");

        KafkaConfig kafkaConfig = new KafkaConfig(properties);
        kafkaServer = new KafkaServerStartable(kafkaConfig);
        kafkaServer.startup();

        TimeUnit.SECONDS.sleep(WAIT_SECONDS);
    }
 
Example #17
Source File: EmbeddedKafka.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();

    this.kafkaConfig = kafkaConfig;
    this.zookeeperConfig = zookeeperConfig;

    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
    this.zkServer = new ZooKeeperServer();
}
 
Example #18
Source File: EmbeddedKafka.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #19
Source File: EmbeddedKafka.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #20
Source File: EmbeddedKafka.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #21
Source File: KafkaOperatorTestBase.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public static void startKafkaServer(int clusterid, int brokerid)
{
  Properties props = new Properties();
  props.setProperty("broker.id", "" + clusterid * 10 + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");

  broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid].startup();

}
 
Example #22
Source File: KafkaOperatorTestBase.java    From attic-apex-malhar with Apache License 2.0 5 votes vote down vote up
public static void startKafkaServer(int clusterid, int brokerid)
{
  Properties props = new Properties();
  props.setProperty("broker.id", "" + clusterid * 10 + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");

  broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid].startup();

}
 
Example #23
Source File: EmbeddedKafka.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #24
Source File: EmbeddedKafka.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #25
Source File: EmbeddedKafka.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #26
Source File: KafkaLocal.java    From sqoop-on-spark with Apache License 2.0 5 votes vote down vote up
public KafkaLocal(Properties kafkaProperties) throws IOException,
        InterruptedException{
  kafkaConfig = new KafkaConfig(kafkaProperties);

  //start local kafka broker
  kafka = new KafkaServerStartable(kafkaConfig);
}
 
Example #27
Source File: EmbeddedKafka.java    From mercury with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

    try (InputStream stream = EmbeddedKafka.class.getResourceAsStream("/kafka.properties")) {
        if (stream == null) {
            throw new IOException("kafka.properties is not available as resource");
        }
        Properties p = new Properties();
        p.load(stream);

        String dir = p.getProperty("log.dirs");
        if (dir != null) {
            File reset = new File(dir);
            if (reset.exists() && reset.isDirectory()) {
                Utility.getInstance().cleanupDir(reset);
                log.info("Clean up transient Kafka working directory at {}", dir);
            }
        }
        kafka = new KafkaServerStartable(new KafkaConfig(p));
        kafka.startup();
        Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));

    } catch (IOException e) {
        log.error("Unable to start Kafka kafka - {}", e.getMessage());
        System.exit(-1);
    }
}
 
Example #28
Source File: LocalKafka.java    From beam with Apache License 2.0 5 votes vote down vote up
LocalKafka(int kafkaPort, int zookeeperPort) throws Exception {
  Properties kafkaProperties = new Properties();
  kafkaProperties.setProperty("port", String.valueOf(kafkaPort));
  kafkaProperties.setProperty("zookeeper.connect", String.format("localhost:%s", zookeeperPort));
  kafkaProperties.setProperty("offsets.topic.replication.factor", "1");
  kafkaProperties.setProperty("log.dir", Files.createTempDirectory("kafka-log-").toString());
  server = new KafkaServerStartable(KafkaConfig.fromProps(kafkaProperties));
}
 
Example #29
Source File: KafkaServer.java    From hbase-connect-kafka with Apache License 2.0 5 votes vote down vote up
public KafkaServer(int zookeeperPort, int kafkaBrokerPort) {
    try {
        Preconditions.checkArgument(zookeeperPort > 0);
        Preconditions.checkArgument(kafkaBrokerPort > 0);
        this.zookeeperPort = zookeeperPort;
        this.brokerPort = kafkaBrokerPort;
       this.logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/hbase-cdc-kafka-" + brokerPort);

       KafkaConfig config = buildKafkaConfig(zookeeperPort);
        kafka = new KafkaServerStartable(config);
        kafka.startup();
    } catch (Exception ex) {
        throw new RuntimeException("Could not start test broker", ex);
    }
}
 
Example #30
Source File: KafkaStarterUtils.java    From uReplicator with Apache License 2.0 5 votes vote down vote up
public static void stopServer(KafkaServerStartable serverStartable) {
  serverStartable.shutdown();
  if (StringUtils.isNotEmpty(kafkaDataDir)) {
    try {
      FileUtils.deleteDirectory(new File(kafkaDataDir));
    } catch (IOException e) {
      LOGGER.warn("Caught exception while stopping kafka", e);
    }
  }
}