kafka.server.KafkaConfig Java Examples

The following examples show how to use kafka.server.KafkaConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SASLClusterTestHarness.java    From kcache with Apache License 2.0 7 votes vote down vote up
@Override
protected KafkaConfig getKafkaConfig(int brokerId) {
    final Option<File> trustStoreFileOption = Option.apply(null);
    final Option<SecurityProtocol> saslInterBrokerSecurityProtocol =
        Option.apply(SecurityProtocol.SASL_PLAINTEXT);
    Properties props = TestUtils.createBrokerConfig(
        brokerId, zkConnect, false, false, TestUtils.RandomPort(), saslInterBrokerSecurityProtocol,
        trustStoreFileOption, EMPTY_SASL_PROPERTIES, false, true, TestUtils.RandomPort(),
        false, TestUtils.RandomPort(),
        false, TestUtils.RandomPort(), Option.<String>empty(), 1, false, 1, (short) 1);

    injectProperties(props);
    props.setProperty("zookeeper.connection.timeout.ms", "30000");
    props.setProperty("sasl.mechanism.inter.broker.protocol", "GSSAPI");
    props.setProperty(SaslConfigs.SASL_ENABLED_MECHANISMS, "GSSAPI");

    return KafkaConfig.fromProps(props);
}
 
Example #2
Source File: KafkaTestServerImpl.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public void start() throws Exception {
    zkServer = new TestingServer(zookeeperPort, logDir);
    ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
    curatorClient = CuratorFrameworkFactory.newClient(zkServer.getConnectString(), retryPolicy);
    curatorClient.start();

    Properties props = new Properties();

    props.setProperty("zookeeper.connect", zkServer.getConnectString());
    props.setProperty("broker.id", "0");
    props.setProperty("port", "" + kafkaPort);
    props.setProperty("log.dirs", logDir.getAbsolutePath());
    props.setProperty("auto.create.topics.enable", "true");

    kafkaServer = new KafkaServerStartable(new KafkaConfig(props));
    kafkaServer.startup();
}
 
Example #3
Source File: EmbeddedKafkaServer.java    From twill with Apache License 2.0 6 votes vote down vote up
private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
  return new KafkaServer(kafkaConfig, new Time() {

    @Override
    public long milliseconds() {
      return System.currentTimeMillis();
    }

    @Override
    public long nanoseconds() {
      return System.nanoTime();
    }

    @Override
    public void sleep(long ms) {
      try {
        Thread.sleep(ms);
      } catch (InterruptedException e) {
        Thread.interrupted();
      }
    }
  });
}
 
Example #4
Source File: KafkaUtilsTest.java    From incubator-samoa with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example #5
Source File: KafkaClusterTestBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private KafkaServer createKafkaServer(int brokerId,String _zkConnectString){
	
	int _brokerId = brokerId;
	int _kafkaServerPort = TestUtils.findFreePort();	
	Properties props = kafka.utils.TestUtils.createBrokerConfig(
         _brokerId,
         _zkConnectString,
         kafka.utils.TestUtils.createBrokerConfig$default$3(),
         kafka.utils.TestUtils.createBrokerConfig$default$4(),
         _kafkaServerPort,
         kafka.utils.TestUtils.createBrokerConfig$default$6(),
         kafka.utils.TestUtils.createBrokerConfig$default$7(),
         kafka.utils.TestUtils.createBrokerConfig$default$8(),
         kafka.utils.TestUtils.createBrokerConfig$default$9(),
         kafka.utils.TestUtils.createBrokerConfig$default$10(),
         kafka.utils.TestUtils.createBrokerConfig$default$11(),
         kafka.utils.TestUtils.createBrokerConfig$default$12(),
         kafka.utils.TestUtils.createBrokerConfig$default$13(),
         kafka.utils.TestUtils.createBrokerConfig$default$14()
         );
     KafkaConfig config = new KafkaConfig(props);
     Time mock = new MockTime();
     KafkaServer _kafkaServer = kafka.utils.TestUtils.createServer(config, mock);
     kafkaBrokerPortList.add(_kafkaServerPort);
     return _kafkaServer;
}
 
Example #6
Source File: KafkaDataServerStartable.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
public void init(Properties props) {
  zkStr = props.getProperty(ZOOKEEPER_CONNECT);
  logDirPath = props.getProperty(LOG_DIRS);

  // Create the ZK nodes for Kafka, if needed
  int indexOfFirstSlash = zkStr.indexOf('/');
  if (indexOfFirstSlash != -1) {
    String bareZkUrl = zkStr.substring(0, indexOfFirstSlash);
    String zkNodePath = zkStr.substring(indexOfFirstSlash);
    ZkClient client = new ZkClient(bareZkUrl);
    client.createPersistent(zkNodePath, true);
    client.close();
  }

  File logDir = new File(logDirPath);
  logDir.mkdirs();

  props.put("zookeeper.session.timeout.ms", "60000");
  KafkaConfig config = new KafkaConfig(props);

  serverStartable = new KafkaServerStartable(config);
}
 
Example #7
Source File: KafkaStormIntegrationTest.java    From incubator-retired-pirk with Apache License 2.0 6 votes vote down vote up
private void startKafka() throws Exception
{
  FileUtils.deleteDirectory(new File(kafkaTmpDir));

  Properties props = new Properties();
  props.setProperty("zookeeper.session.timeout.ms", "100000");
  props.put("advertised.host.name", "localhost");
  props.put("port", 11111);
  // props.put("broker.id", "0");
  props.put("log.dir", kafkaTmpDir);
  props.put("enable.zookeeper", "true");
  props.put("zookeeper.connect", zookeeperLocalCluster.getConnectString());
  KafkaConfig kafkaConfig = KafkaConfig.fromProps(props);
  kafkaLocalBroker = new KafkaServer(kafkaConfig, new SystemTime(), scala.Option.apply("kafkaThread"));
  kafkaLocalBroker.startup();

  zkClient = new ZkClient(zookeeperLocalCluster.getConnectString(), 60000, 60000, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperLocalCluster.getConnectString()), false);
  // ZkUtils zkUtils = ZkUtils.apply(zookeeperLocalCluster.getConnectString(), 60000, 60000, false);
  AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties());
}
 
Example #8
Source File: LocalKafkaServer.java    From Krackle with Apache License 2.0 6 votes vote down vote up
public LocalKafkaServer() throws IOException {

		while (new File(logDir).exists()) {
			FileUtils.deleteDirectory(new File(logDir));
		}

		Properties props = new Properties();
		props.put("broker.id", nodeId);
		props.put("port", port);
		props.put("log.dir", logDir);
		props.put("zookeeper.connect", zkConnect);
		props.put("host.name", "127.0.0.1");
		KafkaConfig conf = new KafkaConfig(props);

                zkUtils = ZkUtils.apply(props.getProperty("zookeeper.connect"),
                          30000,
                          30000,
                          JaasUtils.isZkSecurityEnabled());


		server = new KafkaServerStartable(conf);
		server.startup();
	}
 
Example #9
Source File: KafkaEmbedded.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and starts an embedded Kafka broker.
 *
 * @param config Broker configuration settings.  Used to modify, for example, the listeners
 *               the broker should use.  Note that you cannot change some settings such as
 *               `log.dirs`.
 */
public KafkaEmbedded(final Properties config) throws IOException {
  this.tmpFolder = new TemporaryFolder();
  this.tmpFolder.create();
  this.logDir = tmpFolder.newFolder();
  this.effectiveConfig = effectiveConfigFrom(config, logDir);

  final KafkaConfig kafkaConfig = new KafkaConfig(effectiveConfig, true);
  log.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...",
      logDir, zookeeperConnect());

  kafka = TestUtils.createServer(kafkaConfig, new SystemTime());
  log.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...",
      brokerList(), zookeeperConnect());
}
 
Example #10
Source File: KafkaEmbedded.java    From eagle with Apache License 2.0 6 votes vote down vote up
public KafkaEmbedded(Integer kafkaPort, Integer zookeeperPort) {
    try {
        zk = new ZookeeperEmbedded(zookeeperPort);
        zk.start();

        this.port = null != kafkaPort ? kafkaPort : InstanceSpec.getRandomPort();
        logDir = new File(System.getProperty("java.io.tmpdir"), "kafka/logs/kafka-test-" + kafkaPort);
        FileUtils.deleteQuietly(logDir);

        KafkaConfig config = buildKafkaConfig(zk.getConnectionString());
        kafka = new KafkaServerStartable(config);
        kafka.startup();
    } catch (Exception ex) {
        throw new RuntimeException("Could not start test broker", ex);
    }
}
 
Example #11
Source File: KafkaTestBase.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
public static void startServer() throws RuntimeException {
  if (serverStarted && serverClosed) {
    throw new RuntimeException("Kafka test server has already been closed. Cannot generate Kafka server twice.");
  }

  if (!serverStarted) {
    serverStarted = true;
    zkConnect = TestZKUtils.zookeeperConnect();
    zkServer = new EmbeddedZookeeper(zkConnect);
    zkClient = new ZkClient(zkServer.connectString(), 30000, 30000, ZKStringSerializer$.MODULE$);

    kafkaPort = TestUtils.choosePort();
    Properties props = TestUtils.createBrokerConfig(brokerId, kafkaPort, true);

    KafkaConfig config = new KafkaConfig(props);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
  }
}
 
Example #12
Source File: EmbeddedKafkaInstance.java    From rya with Apache License 2.0 6 votes vote down vote up
/**
 * Starts the Embedded Kafka and Zookeeper Servers.
 * @throws Exception - If an exeption occurs during startup.
 */
protected void startup() throws Exception {
    // Setup the embedded zookeeper
    logger.info("Starting up Embedded Zookeeper...");
    zkServer = new EmbeddedZookeeper();
    zookeperConnect = ZKHOST + ":" + zkServer.port();
    logger.info("Embedded Zookeeper started at: {}", zookeperConnect);

    // setup Broker
    logger.info("Starting up Embedded Kafka...");
    brokerPort = Integer.toString(PortUtils.getRandomFreePort());
    final Properties brokerProps = new Properties();
    brokerProps.setProperty(KafkaConfig$.MODULE$.BrokerIdProp(), "0");
    brokerProps.setProperty(KafkaConfig$.MODULE$.HostNameProp(), BROKERHOST);
    brokerProps.setProperty(KafkaConfig$.MODULE$.PortProp(), brokerPort);
    brokerProps.setProperty(KafkaConfig$.MODULE$.ZkConnectProp(), zookeperConnect);
    brokerProps.setProperty(KafkaConfig$.MODULE$.LogDirsProp(), Files.createTempDirectory(getClass().getSimpleName() + "-").toAbsolutePath().toString());
    brokerProps.setProperty(KafkaConfig$.MODULE$.DeleteTopicEnableProp(), "true");
    final KafkaConfig config = new KafkaConfig(brokerProps);
    final Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);
    logger.info("Embedded Kafka Server started at: {}:{}", BROKERHOST, brokerPort);
}
 
Example #13
Source File: KafkaOperatorTestBase.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
public void startKafkaServer(int clusterid, int brokerid, int defaultPartitions)
{
  // before start, clean the kafka dir if it exists
  FileUtils.deleteQuietly(new File(baseDir, kafkaBaseDir));

  Properties props = new Properties();
  props.setProperty("broker.id", "" + brokerid);
  props.setProperty("log.dirs", new File(baseDir, kafkadir[clusterid][brokerid]).toString());
  props.setProperty("zookeeper.connect", "localhost:" + TEST_ZOOKEEPER_PORT[clusterid]);
  props.setProperty("port", "" + TEST_KAFKA_BROKER_PORT[clusterid][brokerid]);
  props.setProperty("default.replication.factor", "1");
  // set this to 50000 to boost the performance so most test data are in memory before flush to disk
  props.setProperty("log.flush.interval.messages", "50000");
  if (hasMultiPartition) {
    props.setProperty("num.partitions", "2");
  } else {
    props.setProperty("num.partitions", "1");
  }

  broker[clusterid][brokerid] = new KafkaServerStartable(new KafkaConfig(props));
  broker[clusterid][brokerid].startup();

}
 
Example #14
Source File: KafkaEntranceProcessorTest.java    From incubator-samoa with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics        
    AdminUtils.createTopic(zkUtils, TOPIC_OOS, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example #15
Source File: ClusterTestHarness.java    From kcache with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
    zookeeper = new EmbeddedZookeeper();
    zkConnect = String.format("localhost:%d", zookeeper.port());

    configs = new Vector<>();
    servers = new Vector<>();
    for (int i = 0; i < numBrokers; i++) {
        KafkaConfig config = getKafkaConfig(i);
        configs.add(config);

        KafkaServer server = TestUtils.createServer(config, Time.SYSTEM);
        servers.add(server);
    }

    String[] serverUrls = new String[servers.size()];
    ListenerName listenerType = ListenerName.forSecurityProtocol(getSecurityProtocol());
    for (int i = 0; i < servers.size(); i++) {
        serverUrls[i] =
            Utils.formatAddress(
                servers.get(i).config().advertisedListeners().head().host(),
                servers.get(i).boundPort(listenerType)
            );
    }
    bootstrapServers = Utils.join(serverUrls, ",");
}
 
Example #16
Source File: EmbeddedKafkaCluster.java    From common-docker with Apache License 2.0 5 votes vote down vote up
private void startBroker(int brokerId, String zkConnectString) throws IOException {
  if (brokerId < 0) {
    throw new IllegalArgumentException("broker id must not be negative");
  }

  Properties props = TestUtils
      .createBrokerConfig(
          brokerId,
          zkConnectString,
          ENABLE_CONTROLLED_SHUTDOWN,
          ENABLE_DELETE_TOPIC,
          0,
          INTER_BROKER_SECURITY_PROTOCOL,
          this.brokerTrustStoreFile,
          this.brokerSaslProperties,
          ENABLE_PLAINTEXT,
          ENABLE_SASL_PLAINTEXT,
          SASL_PLAINTEXT_PORT,
          ENABLE_SSL,
          SSL_PORT,
          this.enableSASLSSL,
          0,
          Option.<String>empty(),
          1,
          false,
          NUM_PARTITIONS,
          DEFAULT_REPLICATION_FACTOR
      );

  KafkaServer broker = TestUtils.createServer(KafkaConfig.fromProps(props), new MockTime());
  brokersById.put(brokerId, broker);
}
 
Example #17
Source File: EmbeddedKafka.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #18
Source File: KafkaDemoClient.java    From iotplatform with Apache License 2.0 5 votes vote down vote up
private static void startKafkaLocal() throws Exception {
    final File kafkaTmpLogsDir = File.createTempFile("zookeeper", "test");
    if (kafkaTmpLogsDir.delete() && kafkaTmpLogsDir.mkdir()) {
        Properties kafkaProperties = new Properties();
        kafkaProperties.setProperty("host.name", HOSTNAME);
        kafkaProperties.setProperty("port", String.valueOf(KAFKA_PORT));
        kafkaProperties.setProperty("broker.id", String.valueOf(BROKER_ID));
        kafkaProperties.setProperty("zookeeper.connect", ZOOKEEPER_CONNECT);
        kafkaProperties.setProperty("log.dirs", kafkaTmpLogsDir.getAbsolutePath());
        KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);
        KafkaServerStartable kafka = new KafkaServerStartable(kafkaConfig);
        kafka.startup();
    }
}
 
Example #19
Source File: EmbeddedKafka.java    From nifi with Apache License 2.0 5 votes vote down vote up
/**
 * Will create instance of the embedded Kafka server.
 *
 * @param kafkaConfig
 *            Kafka configuration properties
 * @param zookeeperConfig
 *            Zookeeper configuration properties
 */
public EmbeddedKafka(Properties kafkaConfig, Properties zookeeperConfig) {
    this.cleanupKafkaWorkDir();
    this.zookeeperConfig = zookeeperConfig;
    this.kafkaConfig = kafkaConfig;
    this.kafkaPort = this.availablePort();
    this.zookeeperPort = this.availablePort();

    this.kafkaConfig.setProperty("port", String.valueOf(this.kafkaPort));
    this.kafkaConfig.setProperty("zookeeper.connect", "localhost:" + this.zookeeperPort);
    this.zookeeperConfig.setProperty("clientPort", String.valueOf(this.zookeeperPort));
    this.zkServer = new ZooKeeperServer();
    this.kafkaServer = new KafkaServerStartable(new KafkaConfig(kafkaConfig));
}
 
Example #20
Source File: KafkaBrokerTestHarness.java    From common-kafka with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a collection of Kafka Broker configurations based on the number of brokers and zookeeper.
 * @param brokers the number of brokers to create configuration for.
 * @param zookeeperPort the zookeeper port for the brokers to connect to.
 * @param properties properties that should be applied for each broker config.  These properties will be
 *                       honored in favor of any default properties.
 * @return configuration for a collection of brokers.
 * @throws IllegalArgumentException if {@code brokers} is less than 1 or {@code properties} is {@code null}.
 */
public static List<KafkaConfig> getBrokerConfig(int brokers, int zookeeperPort, Properties properties) {
    if (brokers < 1) {
        throw new IllegalArgumentException("Invalid broker count: " + brokers);
    }
    if(properties == null){
        throw new IllegalArgumentException("The 'properties' cannot be 'null'.");
    }

    int ports[] = KafkaTestUtils.getPorts(brokers);

    List<KafkaConfig> configs = new ArrayList<>(brokers);
    for (int i = 0; i < brokers; ++i) {
        Properties props = new Properties();
        props.setProperty(KafkaConfig.ZkConnectProp(), "localhost:" + zookeeperPort);
        props.setProperty(KafkaConfig.BrokerIdProp(), String.valueOf(i + 1));
        props.setProperty(KafkaConfig.AdvertisedHostNameProp(), "localhost");
        props.setProperty(KafkaConfig.HostNameProp(), "localhost");
        props.setProperty(KafkaConfig.PortProp(), String.valueOf(ports[i]));
        props.setProperty(KafkaConfig.LogDirProp(), KafkaTestUtils.getTempDir().getAbsolutePath());
        props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), String.valueOf(1));
        props.setProperty(KafkaConfig.AutoCreateTopicsEnableProp(), String.valueOf(false));
        props.setProperty(KafkaConfig.NumPartitionsProp(), String.valueOf(PARTITIONS_PER_TOPIC));
        props.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp(), String.valueOf(brokers));
        props.setProperty(KafkaConfig.DefaultReplicationFactorProp(), String.valueOf(brokers));
        props.setProperty(KafkaConfig.DeleteTopicEnableProp(), String.valueOf(true));
        props.setProperty(KafkaConfig.OffsetsTopicPartitionsProp(), String.valueOf(PARTITIONS_PER_TOPIC));
        props.setProperty(KafkaConfig.LogIndexSizeMaxBytesProp(), String.valueOf(1024 * 1024));
        props.setProperty(KafkaConfig.LogCleanerEnableProp(), String.valueOf(false));
        props.setProperty(KafkaConfig.GroupInitialRebalanceDelayMsProp(), String.valueOf(100));

        props.putAll(properties);

        configs.add(new KafkaConfig(props));
    }
    return configs;
}
 
Example #21
Source File: KafkaServer.java    From ingestion with Apache License 2.0 5 votes vote down vote up
public void start() throws Exception {

        File dir = Files.createTempDir();
        String dirPath = dir.getAbsolutePath();
        System.out.println("Storing Kafka files in " + dirPath);

        String localhostKafka = System.getProperty("kafka.ip");
        String localhostZookeeper = System.getProperty("zookeeper.ip");
        String portKafka = System.getProperty("kafka.port");
        String portZookeeper = System.getProperty("zookeeper.port");

        Properties properties = new Properties();
        properties.put("broker.id", "0");
        properties.put("host.name", "localhost");
//        properties.put("host.name", localhostKafka);
        properties.put("port", KAFKA_PORT);
        properties.put("log.dir", dirPath);
        properties.put("log.flush.interval.messages", "1");
//        properties.put("zookeeper.connect", "localhost:" + ZookeeperServer.CLIENT_PORT);
        properties.put("zookeeper.connect", localhostZookeeper+":" + ZookeeperServer.CLIENT_PORT);
//        properties.put("zookeeper.connect", "port:" + ZookeeperServer.CLIENT_PORT);
//        Integer clientPort = Integer.parseInt(System.getProperty("zookeeper.port"));
//        properties.put("zookeeper.connect", "localhost:" + clientPort);
        properties.put("replica.socket.timeout.ms", "1500");
        properties.put("auto.create.topics.enable", "true");
        properties.put("num.partitions", "1");

        KafkaConfig kafkaConfig = new KafkaConfig(properties);
        kafkaServer = new KafkaServerStartable(kafkaConfig);
        kafkaServer.startup();

        TimeUnit.SECONDS.sleep(WAIT_SECONDS);
    }
 
Example #22
Source File: BrokerBuilder.java    From ameliant-tools with Apache License 2.0 5 votes vote down vote up
public EmbeddedKafkaBroker build() {
    Properties props = brokerDefinition.getProperties();
    props.setProperty(KafkaConfig.HostNameProp(), brokerDefinition.getHostname());
    props.setProperty(KafkaConfig.PortProp(), Integer.toString(brokerDefinition.getPort()));
    props.setProperty(KafkaConfig.BrokerIdProp(), Integer.toString(brokerDefinition.getBrokerId()));
    props.setProperty(KafkaConfig.ZkConnectProp(), brokerDefinition.getZookeeperConnect());
    props.setProperty(KafkaConfig.NumPartitionsProp(), Integer.toString(brokerDefinition.getNumPartitions()));

    Long logFlushIntervalMessages = brokerDefinition.getLogFlushIntervalMessages();
    if (logFlushIntervalMessages != null) {
        props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), logFlushIntervalMessages.toString());
    }

    return new EmbeddedKafkaBroker(brokerDefinition);
}
 
Example #23
Source File: EmbeddedKafkaServer.java    From twill with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@link KafkaConfig} from the given {@link Properties}. If the {@code "port"} property is missing
 * or is equals to {@code "0"}, a random port will be generated.
 */
private KafkaConfig createKafkaConfig(Properties properties) {
  Properties prop = new Properties();
  prop.putAll(properties);

  String port = prop.getProperty("port");
  if (port == null || "0".equals(port)) {
    int randomPort = Networks.getRandomPort();
    Preconditions.checkState(randomPort > 0, "Failed to get random port.");
    prop.setProperty("port", Integer.toString(randomPort));
  }

  return new KafkaConfig(prop);
}
 
Example #24
Source File: KafkaServer.java    From hbase-connect-kafka with Apache License 2.0 5 votes vote down vote up
private kafka.server.KafkaConfig buildKafkaConfig(int zookeeperPort) throws IOException {
    Properties p = new Properties();
    p.setProperty("zookeeper.connect", TO_LOCAL_URI.apply(zookeeperPort));
    p.setProperty("broker.id", "0");
    p.setProperty("port", Integer.toString(this.brokerPort));
    p.setProperty("log.dirs", logDir.getAbsolutePath());
    p.put("host.name", "localhost");
    p.put("controlled.shutdown.enable", "true");

    return new KafkaConfig(p);
}
 
Example #25
Source File: EmbeddedKafka.java    From mercury with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {

    try (InputStream stream = EmbeddedKafka.class.getResourceAsStream("/kafka.properties")) {
        if (stream == null) {
            throw new IOException("kafka.properties is not available as resource");
        }
        Properties p = new Properties();
        p.load(stream);

        String dir = p.getProperty("log.dirs");
        if (dir != null) {
            File reset = new File(dir);
            if (reset.exists() && reset.isDirectory()) {
                Utility.getInstance().cleanupDir(reset);
                log.info("Clean up transient Kafka working directory at {}", dir);
            }
        }
        kafka = new KafkaServerStartable(new KafkaConfig(p));
        kafka.startup();
        Runtime.getRuntime().addShutdownHook(new Thread(this::shutdown));

    } catch (IOException e) {
        log.error("Unable to start Kafka kafka - {}", e.getMessage());
        System.exit(-1);
    }
}
 
Example #26
Source File: KafkaComponent.java    From metron with Apache License 2.0 5 votes vote down vote up
@Override
public void start() {
  // setup Zookeeper
  zookeeperConnectString = topologyProperties.getProperty(ZKServerComponent.ZOOKEEPER_PROPERTY);

  zkClient = new ZkClient(zookeeperConnectString, ZK_SESSION_TIMEOUT_MS, ZK_CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);

  // setup Broker
  Properties props = TestUtilsWrapper.createBrokerConfig(0, zookeeperConnectString, brokerPort);
  props.setProperty("zookeeper.connection.timeout.ms", Integer.toString(KAFKA_ZOOKEEPER_TIMEOUT_MS));
  KafkaConfig config = new KafkaConfig(props);
  Time mock = new MockTime();
  kafkaServer = TestUtils.createServer(config, mock);

  org.apache.log4j.Level oldLevel = UnitTestHelper.getLog4jLevel(KafkaServer.class);
  UnitTestHelper.setLog4jLevel(KafkaServer.class, org.apache.log4j.Level.OFF);
  // do not proceed until the broker is up
  TestUtilsWrapper.waitUntilBrokerIsRunning(kafkaServer,"Timed out waiting for RunningAsBroker State",100000);

  for(Topic topic : getTopics()) {
    try {
      createTopic(topic.name, topic.numPartitions, KAFKA_PROPAGATE_TIMEOUT_MS);
    } catch (InterruptedException e) {
      throw new RuntimeException("Unable to create topic", e);
    }
  }
  UnitTestHelper.setLog4jLevel(KafkaServer.class, oldLevel);
  if(postStartCallback != null) {
    postStartCallback.apply(this);
  }
}
 
Example #27
Source File: KafkaBrokerTestHarness.java    From common-kafka with Apache License 2.0 5 votes vote down vote up
/**
 * Start only the Kafka brokers and not the Zookeepers.
 *
 * @throws IllegalStateException if already started
 */
public void startKafkaCluster() {
    if((brokers != null) && (!brokers.isEmpty()))
        throw new IllegalStateException("Kafka brokers are already running.");

    brokers = new ArrayList<>(brokerConfigs.size());
    for (KafkaConfig config : brokerConfigs) {
        brokers.add(startBroker(config));
    }
}
 
Example #28
Source File: EphemeralKafkaBroker.java    From kafka-junit with Apache License 2.0 5 votes vote down vote up
private KafkaConfig buildKafkaConfig(String zookeeperQuorum) throws IOException {
    kafkaLogDir = Files.createTempDirectory("kafka_junit");

    Properties props = new Properties();
    props.put("advertised.listeners", "PLAINTEXT://" + LOCALHOST + ":" + kafkaPort);
    props.put("listeners", "PLAINTEXT://0.0.0.0:" + kafkaPort);
    props.put("port", kafkaPort + "");
    props.put("broker.id", "1");
    props.put("log.dirs", kafkaLogDir.toAbsolutePath().toString());
    props.put("zookeeper.connect", zookeeperQuorum);
    props.put("leader.imbalance.check.interval.seconds", "1");
    props.put("offsets.topic.num.partitions", "1");
    props.put("offsets.topic.replication.factor", "1");
    props.put("default.replication.factor", "1");
    props.put("num.partitions", "1");
    props.put("group.min.session.timeout.ms", "100");
    props.put("transaction.state.log.replication.factor", "1");
    props.put("transaction.state.log.min.isr", "1");
    props.put("transaction.state.log.num.partitions", "1");
    props.put("transaction.timeout.ms", "500");

    if (overrideBrokerProperties != null) {
        props.putAll(overrideBrokerProperties);
    }

    return new KafkaConfig(props);
}
 
Example #29
Source File: KafkaLocal.java    From sqoop-on-spark with Apache License 2.0 5 votes vote down vote up
public KafkaLocal(Properties kafkaProperties) throws IOException,
        InterruptedException{
  kafkaConfig = new KafkaConfig(kafkaProperties);

  //start local kafka broker
  kafka = new KafkaServerStartable(kafkaConfig);
}
 
Example #30
Source File: CruiseControlMetricsReporterTest.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Override
public Properties overridingProps() {
  Properties props = new Properties();
  int port = CCKafkaTestUtils.findLocalPort();
  props.setProperty(CommonClientConfigs.METRIC_REPORTER_CLASSES_CONFIG, CruiseControlMetricsReporter.class.getName());
  props.setProperty(KafkaConfig.ListenersProp(), "PLAINTEXT://127.0.0.1:" + port);
  props.setProperty(CruiseControlMetricsReporterConfig.config(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG),
                    "127.0.0.1:" + port);
  props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_REPORTER_INTERVAL_MS_CONFIG, "100");
  props.setProperty(CruiseControlMetricsReporterConfig.CRUISE_CONTROL_METRICS_TOPIC_CONFIG, TOPIC);
  props.setProperty(KafkaConfig.LogFlushIntervalMessagesProp(), "1");
  props.setProperty(KafkaConfig.OffsetsTopicReplicationFactorProp(), "1");
  props.setProperty(KafkaConfig.DefaultReplicationFactorProp(), "2");
  return props;
}