kafka.common.KafkaException Java Examples

The following examples show how to use kafka.common.KafkaException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaAdminClient.java    From data-highway with Apache License 2.0 6 votes vote down vote up 
public void createTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = new Properties(defaultTopicConfig);
  topicConfig.setProperty(LEADER_THROTTLED_REPLICAS, WILDCARD);
  topicConfig.setProperty(FOLLOWER_THROTTLED_REPLICAS, WILDCARD);
  RoadType roadType = ofNullable(road.getType()).orElse(RoadType.NORMAL);
  switch (roadType) {
  case NORMAL:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_DELETE);
    break;
  case COMPACT:
    topicConfig.setProperty(TopicConfig.CLEANUP_POLICY_CONFIG, TopicConfig.CLEANUP_POLICY_COMPACT);
    break;
  default:
    throw new KafkaException("Unhandled road type \"" + road.getType().name() + "\"");
  }
  AdminUtils
      .createTopic(zkUtils, road.getTopicName(), partitions, replicationFactor, topicConfig, DEFAULT_RACK_AWARE_MODE);
  log.info("Created {} topic {}", roadType, road.getTopicName());
}
Example #2
Source File: KafkaAdminClient.java    From data-highway with Apache License 2.0 5 votes vote down vote up 
public void checkAndUpdateTopic(KafkaRoad road) throws KafkaException {
  Properties topicConfig = AdminUtils.fetchEntityConfig(zkUtils, ConfigType.Topic(), road.getTopicName());
  boolean modified = false;
  modified |= checkAndUpdateThrottledReplicas(topicConfig, LEADER_THROTTLED_REPLICAS);
  modified |= checkAndUpdateThrottledReplicas(topicConfig, FOLLOWER_THROTTLED_REPLICAS);
  if (modified) {
    AdminUtils.changeTopicConfig(zkUtils, road.getTopicName(), topicConfig);
    log.info("Updated topic {}", road.getTopicName());
  }
}
Example #3
Source File: Props.java    From HiveKa with Apache License 2.0 5 votes vote down vote up 
/**
 * build props from a list of strings and interpret them as
 * key, value, key, value,....
 *
 * @param args
 * @return props
 */
@SuppressWarnings("unchecked")
public static Props of(String... args) {
  if (args.length % 2 != 0)
    throw new KafkaException(
        "Must have an equal number of keys and values.");
  Map<String, String> vals = new HashMap<String, String>(args.length / 2);
  for (int i = 0; i < args.length; i += 2)
    vals.put(args[i], args[i + 1]);
  return new Props(vals);
}
Example #4
Source File: KafkaConsumer.java    From jstorm with Apache License 2.0 4 votes vote down vote up 
public ByteBufferMessageSet fetchMessages(int partition, long offset) throws IOException {

        String topic = config.topic;
        FetchRequest req = new FetchRequestBuilder().clientId(config.clientId).addFetch(topic, partition, offset, config.fetchMaxBytes)
                .maxWait(config.fetchWaitMaxMs).build();
        FetchResponse fetchResponse = null;
        SimpleConsumer simpleConsumer = null;
        try {
            simpleConsumer = findLeaderConsumer(partition);
            if (simpleConsumer == null) {
                // LOG.error(message);
                return null;
            }
            fetchResponse = simpleConsumer.fetch(req);
        } catch (Exception e) {
            if (e instanceof ConnectException || e instanceof SocketTimeoutException || e instanceof IOException
                    || e instanceof UnresolvedAddressException) {
                LOG.warn("Network error when fetching messages:", e);
                if (simpleConsumer != null) {
                    String host = simpleConsumer.host();
                    int port = simpleConsumer.port();
                    simpleConsumer = null;
                    throw new KafkaException("Network error when fetching messages: " + host + ":" + port + " , " + e.getMessage(), e);
                }

            } else {
                throw new RuntimeException(e);
            }
        }
        if (fetchResponse.hasError()) {
            short code = fetchResponse.errorCode(topic, partition);
            if (code == ErrorMapping.OffsetOutOfRangeCode() && config.resetOffsetIfOutOfRange) {
                long startOffset = getOffset(topic, partition, config.startOffsetTime);
                offset = startOffset;
            }
            if(leaderBroker != null) {
                LOG.error("fetch data from kafka topic[" + config.topic + "] host[" + leaderBroker.host() + ":" + leaderBroker.port() + "] partition["
                    + partition + "] error:" + code);
            }else {
                
            }
            return null;
        } else {
            ByteBufferMessageSet msgs = fetchResponse.messageSet(topic, partition);
            return msgs;
        }
    }
Example #5
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #6
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #7
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #8
Source File: TestKafkaCheckpointManagerJava.java    From samza with Apache License 2.0 4 votes vote down vote up 
public KafkaCheckpointLogKey fromBytes(byte[] bytes) {
  throw new KafkaException("exception");
}
Example #9
Source File: TestKafkaCheckpointManagerJava.java    From samza with Apache License 2.0 4 votes vote down vote up 
public Checkpoint fromBytes(byte[] bytes) {
  throw new KafkaException("exception");
}
Example #10
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #11
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #12
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #13
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	LOG.info("Starting broker with id {}", brokerId);
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", "30000");
	kafkaProperties.put("zookeeper.connection.timeout.ms", "30000");
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", Integer.toString(proxy.getLocalPort()));
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #14
Source File: KafkaTestEnvironmentImpl.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #15
Source File: KafkaTestEnvironmentImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #16
Source File: KafkaTestEnvironmentImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #17
Source File: KafkaTestEnvironmentImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("transaction.max.timeout.ms", Integer.toString(1000 * 60 * 60 * 2)); // 2hours

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, Time.SYSTEM, stringNone, new ArraySeq<KafkaMetricsReporter>(0));
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #18
Source File: KafkaTestEnvironmentImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	LOG.info("Starting broker with id {}", brokerId);
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", "30000");
	kafkaProperties.put("zookeeper.connection.timeout.ms", "30000");
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", Integer.toString(proxy.getLocalPort()));
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #19
Source File: KafkaTestEnvironmentImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Copied from com.github.sakserv.minicluster.KafkaLocalBrokerIntegrationTest (ASL licensed).
 */
protected KafkaServer getKafkaServer(int brokerId, File tmpFolder) throws Exception {
	Properties kafkaProperties = new Properties();

	// properties have to be Strings
	kafkaProperties.put("advertised.host.name", KAFKA_HOST);
	kafkaProperties.put("broker.id", Integer.toString(brokerId));
	kafkaProperties.put("log.dir", tmpFolder.toString());
	kafkaProperties.put("zookeeper.connect", zookeeperConnectionString);
	kafkaProperties.put("message.max.bytes", String.valueOf(50 * 1024 * 1024));
	kafkaProperties.put("replica.fetch.max.bytes", String.valueOf(50 * 1024 * 1024));

	// for CI stability, increase zookeeper session timeout
	kafkaProperties.put("zookeeper.session.timeout.ms", zkTimeout);
	kafkaProperties.put("zookeeper.connection.timeout.ms", zkTimeout);
	if (config.getKafkaServerProperties() != null) {
		kafkaProperties.putAll(config.getKafkaServerProperties());
	}

	final int numTries = 5;

	for (int i = 1; i <= numTries; i++) {
		int kafkaPort = NetUtils.getAvailablePort();
		kafkaProperties.put("port", Integer.toString(kafkaPort));

		if (config.isHideKafkaBehindProxy()) {
			NetworkFailuresProxy proxy = createProxy(KAFKA_HOST, kafkaPort);
			kafkaProperties.put("advertised.port", proxy.getLocalPort());
		}

		//to support secure kafka cluster
		if (config.isSecureMode()) {
			LOG.info("Adding Kafka secure configurations");
			kafkaProperties.put("listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.put("advertised.listeners", "SASL_PLAINTEXT://" + KAFKA_HOST + ":" + kafkaPort);
			kafkaProperties.putAll(getSecureProperties());
		}

		KafkaConfig kafkaConfig = new KafkaConfig(kafkaProperties);

		try {
			scala.Option<String> stringNone = scala.Option.apply(null);
			KafkaServer server = new KafkaServer(kafkaConfig, SystemTime$.MODULE$, stringNone);
			server.startup();
			return server;
		}
		catch (KafkaException e) {
			if (e.getCause() instanceof BindException) {
				// port conflict, retry...
				LOG.info("Port conflict when starting Kafka Broker. Retrying...");
			}
			else {
				throw e;
			}
		}
	}

	throw new Exception("Could not start Kafka after " + numTries + " retries due to port conflicts.");
}
Example #20
Source File: KafkaBrokerMonitor.java    From data-highway with Apache License 2.0 4 votes vote down vote up 
@Scheduled(fixedDelayString = "${kafka.broker.monitor.fixed.delay:60000}")
public void checkAndUpdateBrokers() throws KafkaException {
  adminClient.getBrokerIds().forEach(this::checkAndUpdateBroker);
}