org.apache.kafka.common.utils.SystemTime Java Examples

The following examples show how to use org.apache.kafka.common.utils.SystemTime. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IgniteSinkConnectorTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
    kafkaBroker = new TestKafkaBroker();

    for (String topic : TOPICS)
        kafkaBroker.createTopic(topic, PARTITIONS, REPLICATION_FACTOR);

    Map<String, String> props = makeWorkerProps();
    WorkerConfig workerCfg = new StandaloneConfig(props);

    OffsetBackingStore offBackingStore = mock(OffsetBackingStore.class);
    offBackingStore.configure(workerCfg);

    worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore);
    worker.start();

    herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg));
    herder.start();
}
 
Example #2
Source File: KafkaEmbedded.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
/**
 * Creates and starts an embedded Kafka broker.
 *
 * @param config Broker configuration settings.  Used to modify, for example, the listeners
 *               the broker should use.  Note that you cannot change some settings such as
 *               `log.dirs`.
 */
public KafkaEmbedded(final Properties config) throws IOException {
  this.tmpFolder = new TemporaryFolder();
  this.tmpFolder.create();
  this.logDir = tmpFolder.newFolder();
  this.effectiveConfig = effectiveConfigFrom(config, logDir);

  final KafkaConfig kafkaConfig = new KafkaConfig(effectiveConfig, true);
  log.debug("Starting embedded Kafka broker (with log.dirs={} and ZK ensemble at {}) ...",
      logDir, zookeeperConnect());

  kafka = TestUtils.createServer(kafkaConfig, new SystemTime());
  log.debug("Startup of embedded Kafka broker at {} completed (with ZK ensemble at {}) ...",
      brokerList(), zookeeperConnect());
}
 
Example #3
Source File: MetricCollectors.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
public static void initialize() {
  MetricConfig metricConfig = new MetricConfig()
      .samples(100)
      .timeWindow(
          1000,
          TimeUnit.MILLISECONDS
      );
  List<MetricsReporter> reporters = new ArrayList<>();
  reporters.add(new JmxReporter("io.confluent.ksql.metrics"));
  // Replace all static contents other than Time to ensure they are cleaned for tests that are
  // not aware of the need to initialize/cleanup this test, in case test processes are reused.
  // Tests aware of the class clean everything up properly to get the state into a clean state,
  // a full, fresh instantiation here ensures something like KsqlEngineMetricsTest running after
  // another test that used MetricsCollector without running cleanUp will behave correctly.
  metrics = new Metrics(metricConfig, reporters, new SystemTime());
  collectorMap = new ConcurrentHashMap<>();
}
 
Example #4
Source File: ConsumerService.java    From kafka-monitor with Apache License 2.0 6 votes vote down vote up
public ConsumerService() {

        thread = new Thread(() -> {
            consumer();
        }, name + "consumer-service");

        Properties props = ConfigService.getKafkaConsumerConf();
        MONITOR_TOPIC = ConfigService.monitorConfig.getMonitorTopic();

        consumer = new MonitorConsumer(MONITOR_TOPIC, props);

        MetricConfig metricConfig = new MetricConfig().samples(60).timeWindow(1000, TimeUnit.MILLISECONDS);
        List<MetricsReporter> reporterList = new ArrayList<>();
        reporterList.add(new JmxReporter("kmf.services"));
        Metrics metrics = new Metrics(metricConfig, reporterList, new SystemTime());
        Map<String, String> tags = new HashMap<>();
        tags.put("name", "monitor");
        sensor = new ConsumerMetrics(metrics, tags);
    }
 
Example #5
Source File: IgniteSourceConnectorTest.java    From ignite with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
    kafkaBroker = new TestKafkaBroker();

    Map<String, String> props = makeWorkerProps();
    WorkerConfig workerCfg = new StandaloneConfig(props);

    MemoryOffsetBackingStore offBackingStore = new MemoryOffsetBackingStore();
    offBackingStore.configure(workerCfg);

    worker = new Worker(WORKER_ID, new SystemTime(), new Plugins(props), workerCfg, offBackingStore);
    worker.start();

    herder = new StandaloneHerder(worker, ConnectUtils.lookupKafkaClusterId(workerCfg));
    herder.start();
}
 
Example #6
Source File: ClusterTopicManipulationService.java    From kafka-monitor with Apache License 2.0 6 votes vote down vote up
public ClusterTopicManipulationService(String name, AdminClient adminClient) {
  LOGGER.info("ClusterTopicManipulationService constructor initiated {}", this.getClass().getName());

  _isOngoingTopicCreationDone = true;
  _isOngoingTopicDeletionDone = true;
  _adminClient = adminClient;
  _executor = Executors.newSingleThreadScheduledExecutor();
  _reportIntervalSecond = Duration.ofSeconds(1);
  _running = new AtomicBoolean(false);
  _configDefinedServiceName = name;
  // TODO: instantiate a new instance of ClusterTopicManipulationMetrics(..) here.

  MetricConfig metricConfig = new MetricConfig().samples(60).timeWindow(1000, TimeUnit.MILLISECONDS);
  List<MetricsReporter> reporters = new ArrayList<>();
  reporters.add(new JmxReporter(Service.JMX_PREFIX));
  Metrics metrics = new Metrics(metricConfig, reporters, new SystemTime());

  Map<String, String> tags = new HashMap<>();
  tags.put("name", name);
  _clusterTopicManipulationMetrics = new ClusterTopicManipulationMetrics(metrics, tags);
}
 
Example #7
Source File: ConnectEmbedded.java    From hello-kafka-streams with Apache License 2.0 6 votes vote down vote up
public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception {
    Time time = new SystemTime();
    DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig));

    KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
    offsetBackingStore.configure(config);

    //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random
    String workerId = UUID.randomUUID().toString();
    worker = new Worker(workerId, time, config, offsetBackingStore);

    StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter());
    statusBackingStore.configure(config);

    ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter());
    configBackingStore.configure(config);

    //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything
    herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, "");
    this.connectorConfigs = connectorConfigs;

    shutdownHook = new ShutdownHook();
}
 
Example #8
Source File: XinfraMonitor.java    From kafka-monitor with Apache License 2.0 5 votes vote down vote up
/**
 * XinfraMonitor constructor creates apps and services for each of the individual clusters (properties) that's passed in.
 * For example, if there are 10 clusters to be monitored, then this Constructor will create 10 * num_apps_per_cluster
 * and 10 * num_services_per_cluster.
 * @param allClusterProps the properties of ALL kafka clusters for which apps and services need to be appended.
 * @throws Exception when exception occurs while assigning Apps and Services
 */

@SuppressWarnings({"rawtypes"})
public XinfraMonitor(Map<String, Map> allClusterProps) throws Exception {
  _apps = new ConcurrentHashMap<>();
  _services = new ConcurrentHashMap<>();

  for (Map.Entry<String, Map> clusterProperty : allClusterProps.entrySet()) {
    String name = clusterProperty.getKey();
    Map props = clusterProperty.getValue();
    if (!props.containsKey(XinfraMonitorConstants.CLASS_NAME_CONFIG))
      throw new IllegalArgumentException(name + " is not configured with " + XinfraMonitorConstants.CLASS_NAME_CONFIG);
    String className = (String) props.get(XinfraMonitorConstants.CLASS_NAME_CONFIG);

    Class<?> aClass = Class.forName(className);
    if (App.class.isAssignableFrom(aClass)) {
      App clusterApp = (App) Class.forName(className).getConstructor(Map.class, String.class).newInstance(props, name);
      _apps.put(name, clusterApp);
    } else if (Service.class.isAssignableFrom(aClass)) {
      ServiceFactory serviceFactory = (ServiceFactory) Class.forName(className + XinfraMonitorConstants.FACTORY)
          .getConstructor(Map.class, String.class)
          .newInstance(props, name);
      Service service = serviceFactory.createService();
      _services.put(name, service);
    } else {
      throw new IllegalArgumentException(className + " should implement either " + App.class.getSimpleName() + " or " + Service.class.getSimpleName());
    }
  }
  _executor = Executors.newSingleThreadScheduledExecutor();
  _offlineRunnables = new ConcurrentHashMap<>();
  List<MetricsReporter> reporters = new ArrayList<>();
  reporters.add(new JmxReporter(XinfraMonitorConstants.JMX_PREFIX));
  Metrics metrics = new Metrics(new MetricConfig(), reporters, new SystemTime());
  metrics.addMetric(metrics.metricName("offline-runnable-count", XinfraMonitorConstants.METRIC_GROUP_NAME, "The number of Service/App that are not fully running"),
    (config, now) -> _offlineRunnables.size());
}
 
Example #9
Source File: EmbeddedKafkaBroker.java    From karaf-decanter with Apache License 2.0 5 votes vote down vote up
private KafkaServer startBroker(Properties props) {
    List<KafkaMetricsReporter> kmrList = new ArrayList<>();
    Buffer<KafkaMetricsReporter> metricsList = scala.collection.JavaConversions.asScalaBuffer(kmrList);
    KafkaServer server = new KafkaServer(new KafkaConfig(props), new SystemTime(), Option.<String>empty(), metricsList);
    server.startup();
    return server;
}
 
Example #10
Source File: TestKafkaBroker.java    From ignite with Apache License 2.0 5 votes vote down vote up
/**
 * Sets up test Kafka broker.
 *
 * @throws IOException If failed.
 */
private void setupKafkaServer() throws IOException {
    kafkaCfg = new KafkaConfig(getKafkaConfig());

    kafkaSrv = TestUtils.createServer(kafkaCfg, new SystemTime());

    kafkaSrv.startup();
}
 
Example #11
Source File: ConsumeService.java    From kafka-monitor with Apache License 2.0 5 votes vote down vote up
/**
 * Mainly contains services for three metrics:
 * 1 - ConsumeAvailability metrics
 * 2 - CommitOffsetAvailability metrics
 *   2.1 - commitAvailabilityMetrics records offsets committed upon success. that is, no exception upon callback
 *   2.2 - commitAvailabilityMetrics records offsets commit fail upon failure. that is, exception upon callback
 * 3 - CommitOffsetLatency metrics
 *   3.1 - commitLatencyMetrics records the latency between last successful callback and start of last recorded commit.
 *
 * @param name Name of the Monitor instance
 * @param topicPartitionResult The completable future for topic partition
 * @param consumerFactory Consumer Factory object.
 * @throws ExecutionException when attempting to retrieve the result of a task that aborted by throwing an exception
 * @throws InterruptedException when a thread is waiting, sleeping, or otherwise occupied and the thread is interrupted
 */
public ConsumeService(String name,
                      CompletableFuture<Void> topicPartitionResult,
                      ConsumerFactory consumerFactory)
    throws ExecutionException, InterruptedException {
  _baseConsumer = consumerFactory.baseConsumer();
  _latencySlaMs = consumerFactory.latencySlaMs();
  _name = name;
  _adminClient = consumerFactory.adminClient();
  _running = new AtomicBoolean(false);

  // Returns a new CompletionStage (topicPartitionFuture) which
  // executes the given action - code inside run() - when this stage (topicPartitionResult) completes normally,.
  CompletableFuture<Void> topicPartitionFuture = topicPartitionResult.thenRun(() -> {
    MetricConfig metricConfig = new MetricConfig().samples(60).timeWindow(1000, TimeUnit.MILLISECONDS);
    List<MetricsReporter> reporters = new ArrayList<>();
    reporters.add(new JmxReporter(JMX_PREFIX));
    metrics = new Metrics(metricConfig, reporters, new SystemTime());
    tags = new HashMap<>();
    tags.put(TAGS_NAME, name);
    _topic = consumerFactory.topic();
    _sensors = new ConsumeMetrics(metrics, tags, consumerFactory.latencyPercentileMaxMs(),
        consumerFactory.latencyPercentileGranularityMs());
    _commitLatencyMetrics = new CommitLatencyMetrics(metrics, tags, consumerFactory.latencyPercentileMaxMs(),
        consumerFactory.latencyPercentileGranularityMs());
    _commitAvailabilityMetrics = new CommitAvailabilityMetrics(metrics, tags);
    _consumeThread = new Thread(() -> {
      try {
        consume();
      } catch (Exception e) {
        LOG.error(name + "/ConsumeService failed", e);
      }
    }, name + " consume-service");
    _consumeThread.setDaemon(true);
  });

  // In a blocking fashion, waits for this topicPartitionFuture to complete, and then returns its result.
  topicPartitionFuture.get();
}
 
Example #12
Source File: AbstractAuditor.java    From li-apache-kafka-clients with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * Construct the abstract auditor.
 */
public AbstractAuditor() {
  super();
  this.setUncaughtExceptionHandler((t, e) -> LOG.error("Auditor died to unhandled exception", e));
  this.setDaemon(true);
  _time = new SystemTime();
}
 
Example #13
Source File: JkesDocumentDeleter.java    From jkes with Apache License 2.0 5 votes vote down vote up
JkesDocumentDeleter(
        JestClient client,
        boolean ignoreSchema,
        Set<String> ignoreSchemaTopics,
        String versionType,
        long flushTimeoutMs,
        int maxBufferedRecords,
        int maxInFlightRequests,
        int batchSize,
        long lingerMs,
        int maxRetries,
        long retryBackoffMs
) {
  this.client = client;
  this.ignoreSchema = ignoreSchema;
  this.ignoreSchemaTopics = ignoreSchemaTopics;
  this.versionType = versionType;
  this.flushTimeoutMs = flushTimeoutMs;

  bulkProcessor = new BulkProcessor<>(
          new SystemTime(),
          new BulkDeletingClient(client),
          maxBufferedRecords,
          maxInFlightRequests,
          batchSize,
          lingerMs,
          maxRetries,
          retryBackoffMs
  );

  existingMappings = new HashSet<>();
}
 
Example #14
Source File: ExecutionTaskManagerTest.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testStateChangeSequences() {
  TopicPartition tp = new TopicPartition("topic", 0);
  ExecutionTaskManager taskManager = new ExecutionTaskManager(null, new MetricRegistry(), new SystemTime(),
          new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));

  List<List<ExecutionTask.State>> testSequences = new ArrayList<>();
  // Completed successfully.
  testSequences.add(Arrays.asList(IN_PROGRESS, COMPLETED));
  // Rollback succeeded.
  testSequences.add(Arrays.asList(IN_PROGRESS, ABORTING, ABORTED));
  // Rollback failed.
  testSequences.add(Arrays.asList(IN_PROGRESS, ABORTING, DEAD));
  // Cannot rollback.
  testSequences.add(Arrays.asList(IN_PROGRESS, DEAD));

  ReplicaPlacementInfo r0 = new ReplicaPlacementInfo(0);
  ReplicaPlacementInfo r1 = new ReplicaPlacementInfo(1);
  ReplicaPlacementInfo r2 = new ReplicaPlacementInfo(2);
  for (List<ExecutionTask.State> sequence : testSequences) {
    taskManager.clear();
    // Make sure the proposal does not involve leader movement.
    ExecutionProposal proposal =
        new ExecutionProposal(tp, 10, r2, Arrays.asList(r0, r2), Arrays.asList(r2, r1));

    taskManager.setExecutionModeForTaskTracker(false);
    taskManager.addExecutionProposals(Collections.singletonList(proposal),
                                      Collections.emptySet(),
                                      generateExpectedCluster(proposal, tp),
                                      null);
    taskManager.setRequestedInterBrokerPartitionMovementConcurrency(null);
    taskManager.setRequestedIntraBrokerPartitionMovementConcurrency(null);
    taskManager.setRequestedLeadershipMovementConcurrency(null);
    List<ExecutionTask> tasks = taskManager.getInterBrokerReplicaMovementTasks();
    assertEquals(1, tasks.size());
    ExecutionTask task  = tasks.get(0);
    verifyStateChangeSequence(sequence, task, taskManager);
  }
}
 
Example #15
Source File: KafkaStore.java    From data-highway with Apache License 2.0 5 votes vote down vote up
public KafkaStore(
    String bootstrapServers,
    Serializer<K, V> serializer,
    String topic,
    Collection<StoreUpdateObserver<K, V>> observers) {
  this(bootstrapServers, serializer, topic, observers, new SystemTime(), emptyMap(), emptyMap());
}
 
Example #16
Source File: KafkaStore.java    From data-highway with Apache License 2.0 5 votes vote down vote up
public KafkaStore(
    String bootstrapServers,
    Serializer<K, V> serializer,
    String topic,
    Map<String, Object> additionalProducerProps,
    Map<String, Object> additionalConsumerProps) {
  this(bootstrapServers, serializer, topic, emptyList(), new SystemTime(), additionalProducerProps,
      additionalConsumerProps);
}
 
Example #17
Source File: ProduceService.java    From kafka-monitor with Apache License 2.0 5 votes vote down vote up
public ProduceService() {

        Properties properties = ConfigService.getZkProper();
        zkConnect = properties.getProperty(ZooConfig.HOST);
        MONITOR_TOPIC = ConfigService.monitorConfig.getMonitorTopic();

        produceExecutor = Executors.newScheduledThreadPool(4, r -> new Thread(r, "produce-service"));
        partitionHandlerExecutor = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "partition-change-handler"));
        partitionNum = new AtomicInteger(1);
        currentPartition = new ConcurrentHashMap<>();

        MetricConfig metricConfig = new MetricConfig().samples(60).timeWindow(100, TimeUnit.MILLISECONDS);
        List<MetricsReporter> reporters = new ArrayList<>();
        reporters.add(new JmxReporter("kmf.services"));
        Metrics metrics = new Metrics(metricConfig, reporters, new SystemTime());
        Map<String, String> tags = new HashMap<>();
        tags.put("name", "test");
        produceMetrics = new ProduceMetrics(metrics, tags);

        int existingPartitionCount = Utils.getPartitionNumByTopic(zkConnect, MONITOR_TOPIC);
        if (existingPartitionCount > 0) {
            partitionNum.set(existingPartitionCount);
        }

        initialProducer();

    }
 
Example #18
Source File: GoalOptimizerTest.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testNoPreComputingThread() {
  Properties props = new Properties();
  props.setProperty(MonitorConfig.BOOTSTRAP_SERVERS_CONFIG, "bootstrap.servers");
  props.setProperty(ExecutorConfig.ZOOKEEPER_CONNECT_CONFIG, "connect:1234");
  props.setProperty(AnalyzerConfig.NUM_PROPOSAL_PRECOMPUTE_THREADS_CONFIG, "0");
  props.setProperty(AnalyzerConfig.DEFAULT_GOALS_CONFIG, TestConstants.DEFAULT_GOALS_VALUES);
  KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);

  GoalOptimizer goalOptimizer = new GoalOptimizer(config, EasyMock.mock(LoadMonitor.class), new SystemTime(),
                                                  new MetricRegistry(), EasyMock.mock(Executor.class));
  // Should exit immediately.
  goalOptimizer.run();
}
 
Example #19
Source File: KafkaCruiseControl.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * Construct the Cruise Control
 *
 * @param config the configuration of Cruise Control.
 */
public KafkaCruiseControl(KafkaCruiseControlConfig config, MetricRegistry dropwizardMetricRegistry) {
  _config = config;
  _time = new SystemTime();
  // initialize some of the static state of Kafka Cruise Control;
  ModelUtils.init(config);
  ModelParameters.init(config);

  // Instantiate the components.
  _anomalyDetector = new AnomalyDetector(this, _time, dropwizardMetricRegistry);
  _executor = new Executor(config, _time, dropwizardMetricRegistry, _anomalyDetector);
  _loadMonitor = new LoadMonitor(config, _time, _executor, dropwizardMetricRegistry, KafkaMetricDef.commonMetricDef());
  _goalOptimizerExecutor = Executors.newSingleThreadExecutor(new KafkaCruiseControlThreadFactory("GoalOptimizerExecutor", true, null));
  _goalOptimizer = new GoalOptimizer(config, _loadMonitor, _time, dropwizardMetricRegistry, _executor);
}
 
Example #20
Source File: AnomalyDetector.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * Package private constructor for unit test.
 */
AnomalyDetector(PriorityBlockingQueue<Anomaly> anomalies,
                AdminClient adminClient,
                long anomalyDetectionIntervalMs,
                KafkaCruiseControl kafkaCruiseControl,
                AnomalyNotifier anomalyNotifier,
                GoalViolationDetector goalViolationDetector,
                BrokerFailureDetector brokerFailureDetector,
                MetricAnomalyDetector metricAnomalyDetector,
                DiskFailureDetector diskFailureDetector,
                TopicAnomalyDetector topicAnomalyDetector,
                ScheduledExecutorService detectorScheduler) {
  _anomalies = anomalies;
  _adminClient = adminClient;
  _anomalyDetectionIntervalMsByType = new HashMap<>(KafkaAnomalyType.cachedValues().size() - 1);
  KafkaAnomalyType.cachedValues().stream().filter(type -> type != BROKER_FAILURE)
                  .forEach(type -> _anomalyDetectionIntervalMsByType.put(type, anomalyDetectionIntervalMs));

  _brokerFailureDetectionBackoffMs = anomalyDetectionIntervalMs;
  _anomalyNotifier = anomalyNotifier;
  _goalViolationDetector = goalViolationDetector;
  _brokerFailureDetector = brokerFailureDetector;
  _metricAnomalyDetector = metricAnomalyDetector;
  _diskFailureDetector = diskFailureDetector;
  _topicAnomalyDetector = topicAnomalyDetector;
  _kafkaCruiseControl = kafkaCruiseControl;
  _detectorScheduler = detectorScheduler;
  _shutdown = false;
  _selfHealingGoals = Collections.emptyList();
  _anomalyLoggerExecutor =
      Executors.newSingleThreadScheduledExecutor(new KafkaCruiseControlThreadFactory("AnomalyLogger", true, null));
  _anomalyInProgress = null;
  _numCheckedWithDelay = new AtomicLong();
  _shutdownLock = new Object();
  // Add anomaly detector state
  _anomalyDetectorState = new AnomalyDetectorState(new SystemTime(), new HashMap<>(KafkaAnomalyType.cachedValues().size()), 10, null);
}
 
Example #21
Source File: NewSenderWithSpringTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {


        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);
        kafkaProps.setProperty("host.name", "localhost");
        kafkaProps.setProperty("port", port + "");

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);

    } catch (Exception e) {
    }
}
 
Example #22
Source File: KafkaMessageNewReceiverPoolTest_2.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

    try {

    zkServer = new EmbeddedZookeeper();
    zkConnect = String.format("localhost:%d", zkServer.port());
    ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
            JaasUtils.isZkSecurityEnabled());
    zkClient = zkUtils.zkClient();

    Time mock = new SystemTime();
    final Option<File> noFile = scala.Option.apply(null);
    final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
    final Option<Properties> noPropertiesOption = scala.Option.apply(null);
    final Option<String> noStringOption = scala.Option.apply(null);

    kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
            false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
            false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
            false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

    kafkaProps.setProperty("auto.create.topics.enable", "true");
    kafkaProps.setProperty("num.partitions", "1");
    // We *must* override this to use the port we allocated (Kafka currently
    // allocates one port
    // that it always uses for ZK
    kafkaProps.setProperty("zookeeper.connect", this.zkConnect);

    KafkaConfig config = new KafkaConfig(kafkaProps);
    kafkaServer = TestUtils.createServer(config, mock);

    // create topic
    TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
            new String[]{"--create", "--topic", topic,
                    "--replication-factor", "1", "--partitions", "2"});

    TopicCommand.createTopic(zkUtils, options);

    List<KafkaServer> servers = new ArrayList<KafkaServer>();
    servers.add(kafkaServer);
    TestUtils.waitUntilMetadataIsPropagated(
            scala.collection.JavaConversions.asScalaBuffer(servers), topic,
            0, 5000);
    } catch (Exception e) {
    }
}
 
Example #23
Source File: ReceiverTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {


        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);
        kafkaProps.setProperty("host.name", "localhost");
        kafkaProps.setProperty("port", port + "");

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {
    }
}
 
Example #24
Source File: SenderWithSpringTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {


        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);
        kafkaProps.setProperty("host.name", "localhost");
        kafkaProps.setProperty("port", port + "");

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);

    } catch (Exception e) {
    }
}
 
Example #25
Source File: ReceiverWithSpringTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {


        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);
        kafkaProps.setProperty("host.name", "localhost");
        kafkaProps.setProperty("port", port + "");

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {
    }
}
 
Example #26
Source File: KafkaCommandTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {
        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);

        KafkaConfig config = new KafkaConfig(kafkaProps);
        Time mock = new SystemTime();
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {

    }
}
 
Example #27
Source File: ZookeeperHostsTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {

        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());
        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);
        kafkaProps.setProperty("host.name", "localhost");
        kafkaProps.setProperty("port", port + "");

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {
    }
}
 
Example #28
Source File: KafkaMessageNewReceiverTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

    try {

        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "1"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {

    }
}
 
Example #29
Source File: KafkaMessageNewSenderTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void setUp() throws Exception {

    try {


        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());

        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "4"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {
    }
}
 
Example #30
Source File: ZookeeperBrokersTest.java    From message-queue-client-framework with Apache License 2.0 4 votes vote down vote up
@Before
public void before() {

    try {

        zkServer = new EmbeddedZookeeper();
        zkConnect = String.format("localhost:%d", zkServer.port());
        ZkUtils zkUtils = ZkUtils.apply(zkConnect, 30000, 30000,
                JaasUtils.isZkSecurityEnabled());
        zkClient = zkUtils.zkClient();

        Time mock = new SystemTime();
        final Option<File> noFile = scala.Option.apply(null);
        final Option<SecurityProtocol> noInterBrokerSecurityProtocol = scala.Option.apply(null);
        final Option<Properties> noPropertiesOption = scala.Option.apply(null);
        final Option<String> noStringOption = scala.Option.apply(null);

        kafkaProps = TestUtils.createBrokerConfig(brokerId, zkConnect, false,
                false, port, noInterBrokerSecurityProtocol, noFile, noPropertiesOption, true,
                false, TestUtils.RandomPort(), false, TestUtils.RandomPort(),
                false, TestUtils.RandomPort(), noStringOption, TestUtils.RandomPort());
        kafkaProps.setProperty("auto.create.topics.enable", "true");
        kafkaProps.setProperty("num.partitions", "1");
        // We *must* override this to use the port we allocated (Kafka currently
        // allocates one port
        // that it always uses for ZK
        kafkaProps.setProperty("zookeeper.connect", this.zkConnect);
        kafkaProps.setProperty("host.name", "localhost");
        kafkaProps.setProperty("port", port + "");

        KafkaConfig config = new KafkaConfig(kafkaProps);
        kafkaServer = TestUtils.createServer(config, mock);

        // create topic
        TopicCommand.TopicCommandOptions options = new TopicCommand.TopicCommandOptions(
                new String[]{"--create", "--topic", topic,
                        "--replication-factor", "1", "--partitions", "4"});

        TopicCommand.createTopic(zkUtils, options);

        List<KafkaServer> servers = new ArrayList<KafkaServer>();
        servers.add(kafkaServer);
        TestUtils.waitUntilMetadataIsPropagated(
                scala.collection.JavaConversions.asScalaBuffer(servers), topic,
                0, 5000);
    } catch (Exception e) {
    }
}