Java Code Examples for kafka.server.KafkaConfig

The following examples show how to use kafka.server.KafkaConfig. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: kcache   Source File: SASLClusterTestHarness.java    License: Apache License 2.0 7 votes vote down vote up
@Override
protected KafkaConfig getKafkaConfig(int brokerId) {
    final Option<File> trustStoreFileOption = Option.apply(null);
    final Option<SecurityProtocol> saslInterBrokerSecurityProtocol =
        Option.apply(SecurityProtocol.SASL_PLAINTEXT);
    Properties props = TestUtils.createBrokerConfig(
        brokerId, zkConnect, false, false, TestUtils.RandomPort(), saslInterBrokerSecurityProtocol,
        trustStoreFileOption, EMPTY_SASL_PROPERTIES, false, true, TestUtils.RandomPort(),
        false, TestUtils.RandomPort(),
        false, TestUtils.RandomPort(), Option.<String>empty(), 1, false, 1, (short) 1);

    injectProperties(props);
    props.setProperty("zookeeper.connection.timeout.ms", "30000");
    props.setProperty("sasl.mechanism.inter.broker.protocol", "GSSAPI");
    props.setProperty(SaslConfigs.SASL_ENABLED_MECHANISMS, "GSSAPI");

    return KafkaConfig.fromProps(props);
}
 
Example 2
Source Project: attic-apex-malhar   Source File: KafkaOperatorTestBase.java    License: Apache License 2.0 6 votes vote down vote up
public void startKafkaServer(int clusterid, int brokerid, int defaultPartitions)
{
  // before start, clean the kafka dir if it exists
  FileUtils.deleteQuietly(new File(baseDir, kafkaBaseDir));

  Properties props = new Properties();
  props.setProperty("broker.id", "" + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid][brokerid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid][brokerid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");
  if (hasMultiPartition) {
    props.setProperty("num.partitions", "2");
  } else {
    props.setProperty("num.partitions", "1");
  }

  broker[clusterid][brokerid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid][brokerid].startup();

}
 
Example 3
Source Project: incubator-gobblin   Source File: KafkaClusterTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private KafkaServer createKafkaServer(int brokerId,String _zkConnectString){
	
	int _brokerId = brokerId;
	int _kafkaServerPort = TestUtils.findFreePort();	
	Properties props = kafka.utils.TestUtils.createBrokerConfig(
         _brokerId,
         _zkConnectString,
         kafka.utils.TestUtils.createBrokerConfig$default$3(),
         kafka.utils.TestUtils.createBrokerConfig$default$4(),
         _kafkaServerPort,
         kafka.utils.TestUtils.createBrokerConfig$default$6(),
         kafka.utils.TestUtils.createBrokerConfig$default$7(),
         kafka.utils.TestUtils.createBrokerConfig$default$8(),
         kafka.utils.TestUtils.createBrokerConfig$default$9(),
         kafka.utils.TestUtils.createBrokerConfig$default$10(),
         kafka.utils.TestUtils.createBrokerConfig$default$11(),
         kafka.utils.TestUtils.createBrokerConfig$default$12(),
         kafka.utils.TestUtils.createBrokerConfig$default$13(),
         kafka.utils.TestUtils.createBrokerConfig$default$14()
         );
     KafkaConfig config = new KafkaConfig(props);
     Time mock = new MockTime();
     KafkaServer _kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
     kafkaBrokerPortList.add(_kafkaServerPort);
     return _kafkaServer;
}
 
Example 4
Source Project: incubator-samoa   Source File: KafkaUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example 5
Source Project: eagle   Source File: KafkaTestServerImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void start() throws Exception {
    zkServer = new TestingServer(zookeeperPort, logDir);
    ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
    curatorClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), retryPolicy);
    curatorClient.start();

    Properties props = new Properties();

    props.setProperty("zookeeper.connect", zkServer.getConnectString());
    props.setProperty("broker.id", "0");
    props.setProperty("port", "" + kafkaPort);
    props.setProperty("log.dirs", logDir.getAbsolutePath());
    props.setProperty("auto.create.topics.enable", "true");

    kafkaServer = new KafkaServerStartable(new KafkaConfig(props));
    kafkaServer.startup();
}
 
Example 6
Source Project: incubator-pinot   Source File: KafkaDataServerStartable.java    License: Apache License 2.0 6 votes vote down vote up
public void init(Properties props) {
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  KafkaConfig config = new KafkaConfig(props);

  serverStartable = new KafkaServerStartable(config);
}
 
Example 7
Source Project: eagle   Source File: KafkaEmbedded.java    License: Apache License 2.0 6 votes vote down vote up
public KafkaEmbedded(Integer kafkaPort, Integer zookeeperPort) {
    try {
        zk = new ZookeeperEmbedded(zookeeperPort);
        zk.start();

        this.port = null != kafkaPort ? kafkaPort : InstanceSpec.getRandomPort();
        logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/kafka-test-" + kafkaPort);
        FileUtils.deleteQuietly(logDir);

        KafkaConfig config = buildKafkaConfig(zk.getConnectionString());
        kafka = new KafkaServerStartable(config);
        kafka.startup();
    } catch (Exception ex) {
        throw new RuntimeException("Could not start test broker", ex);
    }
}
 
Example 8
Source Project: incubator-gobblin   Source File: KafkaTestBase.java    License: Apache License 2.0 6 votes vote down vote up
public static void startServer() throws RuntimeException {
  if (serverStarted && serverClosed) {
    throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice.");
  }

  if (!serverStarted) {
    serverStarted = true;
    zkConnect = TestZKUtils.zookeeperConnect();
    zkServer = new EmbeddedZookeeper(zkConnect);
    zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$);

    kafkaPort = TestUtils.choosePort();
    Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true);

    KafkaConfig config = new KafkaConfig(props);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
  }
}
 
Example 9
Source Project: kcache   Source File: ClusterTestHarness.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
    zookeeper = new EmbeddedZookeeper();
    zkConnect = String.format("localhost:%d", zookeeper.port());

    configs = new Vector<>();
    servers = new Vector<>();
    for (int i = 0; i < numBrokers; i++) {
        KafkaConfig config = getKafkaConfig(i);
        configs.add(config);

        KafkaServer server = TestUtils.createServer(config, Time.SYSTEM);
        servers.add(server);
    }

    String[] serverUrls = new String[servers.size()];
    ListenerName listenerType = ListenerName.forSecurityProtocol(getSecurityProtocol());
    for (int i = 0; i < servers.size(); i++) {
        serverUrls[i] =
            Utils.formatAddress(
                servers.get(i).config().advertisedListeners().head().host(),
                servers.get(i).boundPort(listenerType)
            );
    }
    bootstrapServers = Utils.join(serverUrls, ",");
}
 
Example 10
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics        
    AdminUtils.createTopic(zkUtils, TOPIC_OOS, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example 11
Source Project: rya   Source File: EmbeddedKafkaInstance.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Starts the Embedded Kafka and Zookeeper Servers.
 * @throws Exception - If an exeption occurs during startup.
 */
protected void startup() throws Exception {
    // Setup the embedded zookeeper
    logger.info("Starting up Embedded Zookeeper...");
    zkServer = new EmbeddedZookeeper();
    zookeperConnect = ZKHOST + ":" + zkServer.port();
    logger.info("Embedded Zookeeper started at: {}", zookeperConnect);

    // setup Broker
    logger.info("Starting up Embedded Kafka...");
    brokerPort = Integer.toString(PortUtils.getRandomFreePort());
    final Properties brokerProps = new Properties();
    brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0");
    brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST);
    brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort);
    brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect);
    brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString());
    brokerProps.setProperty(KafkaConfig$.MODULE$.DeleteTopicEnableProp(), "true");
    final KafkaConfig config = new KafkaConfig(brokerProps);
    final Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
    logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort);
}
 
Example 12
Source Project: Krackle   Source File: LocalKafkaServer.java    License: Apache License 2.0 6 votes vote down vote up
public LocalKafkaServer() throws IOException {

		while (new File(logDir).exists()) {
			FileUtils.deleteDirectory(new File(logDir));
		}

		Properties props = new Properties();
		props.put("broker.id", nodeId);
		props.put("port", port);
		props.put("log.dir", logDir);
		props.put("zookeeper.connect", zkConnect);
		props.put("host.name", "127.0.0.1");
		KafkaConfig conf = new KafkaConfig(props);

                zkUtils = ZkUtils.apply(props.getProperty("zookeeper.connect"),
                          30000,
                          30000,
                          JaasUtils.isZkSecurityEnabled());


		server = new KafkaServerStartable(conf);
		server.startup();
	}
 
Example 13
private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}
 
Example 14
Source Project: twill   Source File: EmbeddedKafkaServer.java    License: Apache License 2.0 6 votes vote down vote up
private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
  return new KafkaServer(kafkaConfig, new Time() {

    @Override
    public long milliseconds() {
      return System.currentTimeMillis();
    }

    @Override
    public long nanoseconds() {
      return System.nanoTime();
    }

    @Override
    public void sleep(long ms) {
      try {
        Thread.sleep(ms);
      } catch (InterruptedException e) {
        Thread.interrupted();
      }
    }
  });
}
 
Example 15
Source Project: suro   Source File: KafkaServerExternalResource.java    License: Apache License 2.0 5 votes vote down vote up
public String getBrokerListStr() {
    List<String> str = Lists.newArrayList();
    for (KafkaConfig config : configs) {
        str.add(config.hostName() + ":" + config.port());
    }
    return StringUtils.join(str, ",");
}
 
Example 16
Source Project: ignite   Source File: TestKafkaBroker.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets up test Kafka broker.
 *
 * @throws IOException If failed.
 */
private void setupKafkaServer() throws IOException {
    kafkaCfg = new KafkaConfig(getKafkaConfig());

    kafkaSrv = TestUtils.createServer(kafkaCfg, new SystemTime());

    kafkaSrv.startup();
}
 
Example 17
Source Project: nifi   Source File: EmbeddedKafka.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example 18
Source Project: attic-apex-malhar   Source File: KafkaOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public static void startKafkaServer(int clusterid, int brokerid)
{
  Properties props = new Properties();
  props.setProperty("broker.id", "" + clusterid * 10 + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");

  broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid].startup();

}
 
Example 19
private void removeListenersProp(String listenerType) {
  final String current = (String)brokerConfig.get(KafkaConfig.ListenersProp());
  final String replacement = Arrays.stream(current.split(","))
      .filter(part -> !part.startsWith(listenerType + "://"))
      .collect(Collectors.joining(","));
  brokerConfig.put(KafkaConfig.ListenersProp(), replacement);
}
 
Example 20
/**
 * Creates and starts an embedded Kafka broker.
 *
 * @param config Broker configuration settings.  Used to modify, for example, the listeners
 *               the broker should use.  Note that you cannot change some settings such as
 *               `log.dirs`.
 */
public KafkaEmbedded(final Properties config) throws IOException {
  this.tmpFolder = new TemporaryFolder();
  this.tmpFolder.create();
  this.logDir = tmpFolder.newFolder();
  this.effectiveConfig = effectiveConfigFrom(config, logDir);

  final KafkaConfig kafkaConfig = new KafkaConfig(effectiveConfig, true);
  log.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...",
      logDir, zookeeperConnect());

  kafka = TestUtils.createServer(kafkaConfig, new SystemTime());
  log.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...",
      brokerList(), zookeeperConnect());
}
 
Example 21
Source Project: localization_nifi   Source File: EmbeddedKafka.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example 22
Source Project: localization_nifi   Source File: EmbeddedKafka.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example 23
Source Project: kafka-message-tool   Source File: NodesConfigProperties.java    License: MIT License 5 votes vote down vote up
public static Set<String> getClusterPropertiesThatCanDifferBetweenNodes() {

        return new HashSet<>(Arrays.asList(
            KafkaConfig.LogDirsProp(),
            KafkaConfig.ListenersProp(),
            KafkaConfig.BrokerIdProp(),
            KafkaConfig.AdvertisedListenersProp(),
            // zookeeper.connect can differ if kafka cluster is connected do zookeeper cluster,
            // not just single zookeeper node

            KafkaConfig.ZkConnectProp()
        ));
    }
 
Example 24
Source Project: attic-apex-malhar   Source File: KafkaOperatorTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public static void startKafkaServer(int clusterid, int brokerid)
{
  Properties props = new Properties();
  props.setProperty("broker.id", "" + clusterid * 10 + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");

  broker[clusterid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid].startup();

}
 
Example 25
Source Project: kcache   Source File: SSLClusterTestHarness.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected KafkaConfig getKafkaConfig(int brokerId) {
    File trustStoreFile;
    try {
        trustStoreFile = File.createTempFile("SSLClusterTestHarness-truststore", ".jks");
    } catch (IOException ioe) {
        throw new RuntimeException("Unable to create temporary file for the truststore.");
    }
    final Option<File> trustStoreFileOption = Option.apply(trustStoreFile);
    final Option<SecurityProtocol> sslInterBrokerSecurityProtocol = Option.apply(SecurityProtocol.SSL);
    Properties props = TestUtils.createBrokerConfig(
        brokerId, zkConnect, false, false, TestUtils.RandomPort(), sslInterBrokerSecurityProtocol,
        trustStoreFileOption, EMPTY_SASL_PROPERTIES, false, false, TestUtils.RandomPort(),
        true, TestUtils.RandomPort(), false, TestUtils.RandomPort(), Option.<String>empty(), 1, false,
        1, (short) 1);

    // setup client SSL. Needs to happen before the broker is initialized, because the client's cert
    // needs to be added to the broker's trust store.
    Map<String, Object> sslConfigs;
    try {
        this.clientSslConfigs = TestSslUtils.createSslConfig(true, true, Mode.CLIENT,
            trustStoreFile, "client", "localhost");
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    injectProperties(props);
    if (requireSSLClientAuth()) {
        props.setProperty("ssl.client.auth", "required");
    }

    return KafkaConfig.fromProps(props);
}
 
Example 26
Source Project: kcache   Source File: ClusterTestHarness.java    License: Apache License 2.0 5 votes vote down vote up
protected KafkaConfig getKafkaConfig(int brokerId) {

        final Option<java.io.File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        Properties props = TestUtils.createBrokerConfig(
            brokerId,
            zkConnect,
            false,
            false,
            TestUtils.RandomPort(),
            noInterBrokerSecurityProtocol,
            noFile,
            EMPTY_SASL_PROPERTIES,
            true,
            false,
            TestUtils.RandomPort(),
            false,
            TestUtils.RandomPort(),
            false,
            TestUtils.RandomPort(),
            Option.<String>empty(),
            1,
            false,
            1,
            (short) 1
        );
        injectProperties(props);
        return KafkaConfig.fromProps(props);
    }
 
Example 27
Source Project: common-kafka   Source File: KafkaBrokerTestHarness.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Start only the Kafka brokers and not the Zookeepers.
 *
 * @throws IllegalStateException if already started
 */
public void startKafkaCluster() {
    if((brokers != null) && (!brokers.isEmpty()))
        throw new IllegalStateException("Kafka brokers are already running.");

    brokers = new ArrayList<>(brokerConfigs.size());
    for (KafkaConfig config : brokerConfigs) {
        brokers.add(startBroker(config));
    }
}
 
Example 28
Source Project: common-kafka   Source File: KafkaBrokerTestHarness.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a collection of Kafka Broker configurations based on the number of brokers and zookeeper.
 * @param brokers the number of brokers to create configuration for.
 * @param zookeeperPort the zookeeper port for the brokers to connect to.
 * @param properties properties that should be applied for each broker config.  These properties will be
 *                       honored in favor of any default properties.
 * @return configuration for a collection of brokers.
 * @throws IllegalArgumentException if {@code brokers} is less than 1 or {@code properties} is {@code null}.
 */
public static List<KafkaConfig> getBrokerConfig(int brokers, int zookeeperPort, Properties properties) {
    if (brokers < 1) {
        throw new IllegalArgumentException("Invalid broker count: " + brokers);
    }
    if(properties == null){
        throw new IllegalArgumentException("The 'properties' cannot be 'null'.");
    }

    int ports[] = KafkaTestUtils.getPorts(brokers);

    List<KafkaConfig> configs = new ArrayList<>(brokers);
    for (int i = 0; i < brokers; ++i) {
        Properties props = new Properties();
        props.setProperty(KafkaConfig.ZkConnectProp(), "localhost:" + zookeeperPort);
        props.setProperty(KafkaConfig.BrokerIdProp(), String.valueOf(i + 1));
        props.setProperty(KafkaConfig.AdvertisedHostNameProp(), "localhost");
        props.setProperty(KafkaConfig.HostNameProp(), "localhost");
        props.setProperty(KafkaConfig.PortProp(), String.valueOf(ports[i]));
        props.setProperty(KafkaConfig.LogDirProp(), KafkaTestUtils.getTempDir().getAbsolutePath());
        props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), String.valueOf(1));
        props.setProperty(KafkaConfig.AutoCreateTopicsEnableProp(), String.valueOf(false));
        props.setProperty(KafkaConfig.NumPartitionsProp(), String.valueOf(PARTITIONS_PER_TOPIC));
        props.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp(), String.valueOf(brokers));
        props.setProperty(KafkaConfig.DefaultReplicationFactorProp(), String.valueOf(brokers));
        props.setProperty(KafkaConfig.DeleteTopicEnableProp(), String.valueOf(true));
        props.setProperty(KafkaConfig.OffsetsTopicPartitionsProp(), String.valueOf(PARTITIONS_PER_TOPIC));
        props.setProperty(KafkaConfig.LogIndexSizeMaxBytesProp(), String.valueOf(1024 * 1024));
        props.setProperty(KafkaConfig.LogCleanerEnableProp(), String.valueOf(false));
        props.setProperty(KafkaConfig.GroupInitialRebalanceDelayMsProp(), String.valueOf(100));

        props.putAll(properties);

        configs.add(new KafkaConfig(props));
    }
    return configs;
}
 
Example 29
Source Project: mercury   Source File: EmbeddedKafka.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

    try (InputStream stream = EmbeddedKafka.class.getResourceAsStream("/kafka.properties")) {
        if (stream == null) {
            throw new IOException("kafka.properties is not available as resource");
        }
        Properties p = new Properties();
        p.load(stream);

        String dir = p.getProperty("log.dirs");
        if (dir != null) {
            File reset = new File(dir);
            if (reset.exists() && reset.isDirectory()) {
                Utility.getInstance().cleanupDir(reset);
                log.info("Clean up transient Kafka working directory at {}", dir);
            }
        }
        kafka = new KafkaServerStartable(new KafkaConfig(p));
        kafka.startup();
        Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));

    } catch (IOException e) {
        log.error("Unable to start Kafka kafka - {}", e.getMessage());
        System.exit(-1);
    }
}
 
Example 30
Source Project: ingestion   Source File: KafkaServer.java    License: Apache License 2.0 5 votes vote down vote up
public void start() throws Exception {

        File dir = Files.createTempDir();
        String dirPath = dir.getAbsolutePath();
        System.out.println("Storing Kafka files in " + dirPath);

        String localhostKafka = System.getProperty("kafka.ip");
        String localhostZookeeper = System.getProperty("zookeeper.ip");
        String portKafka = System.getProperty("kafka.port");
        String portZookeeper = System.getProperty("zookeeper.port");

        Properties properties = new Properties();
        properties.put("broker.id", "0");
        properties.put("host.name", "localhost");
//        properties.put("host.name", localhostKafka);
        properties.put("port", KAFKA_PORT);
        properties.put("log.dir", dirPath);
        properties.put("log.flush.interval.messages", "1");
//        properties.put("zookeeper.connect", "localhost:" + ZookeeperServer.CLIENT_PORT);
        properties.put("zookeeper.connect", localhostZookeeper+":" + ZookeeperServer.CLIENT_PORT);
//        properties.put("zookeeper.connect", "port:" + ZookeeperServer.CLIENT_PORT);
//        Integer clientPort = Integer.parseInt(System.getProperty("zookeeper.port"));
//        properties.put("zookeeper.connect", "localhost:" + clientPort);
        properties.put("replica.socket.timeout.ms", "1500");
        properties.put("auto.create.topics.enable", "true");
        properties.put("num.partitions", "1");

        KafkaConfig kafkaConfig = new KafkaConfig(properties);
        kafkaServer = new KafkaServerStartable(kafkaConfig);
        kafkaServer.startup();

        TimeUnit.SECONDS.sleep(WAIT_SECONDS);
    }