org.apache.kafka.common.utils.Time Java Examples

The following examples show how to use org.apache.kafka.common.utils.Time. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaMetricsTest.java    From micrometer with Apache License 2.0 7 votes vote down vote up
@Test
void shouldKeepMetersWhenMetricsDoNotChange() {
    //Given
    Supplier<Map<MetricName, ? extends Metric>> supplier = () -> {
        MetricName metricName = new MetricName("a", "b", "c", new LinkedHashMap<>());
        KafkaMetric metric = new KafkaMetric(this, metricName, new Value(), new MetricConfig(), Time.SYSTEM);
        return Collections.singletonMap(metricName, metric);
    };
    kafkaMetrics = new KafkaMetrics(supplier);
    MeterRegistry registry = new SimpleMeterRegistry();

    kafkaMetrics.bindTo(registry);
    assertThat(registry.getMeters()).hasSize(1);

    kafkaMetrics.checkAndBindMetrics(registry);
    assertThat(registry.getMeters()).hasSize(1);
}
 
Example #2
Source File: Compatibility.java    From apicurio-registry with Apache License 2.0 6 votes vote down vote up
static Worker createWorker(String workerId,
                           Time time,
                           Plugins plugins,
                           WorkerConfig config,
                           OffsetBackingStore offsetBackingStore,
                           Object connectorClientConfigOverridePolicy) throws ConnectException {

    if (CTR_WORKER_22 == null) {
        return new Worker(workerId, time, plugins, config, offsetBackingStore, (ConnectorClientConfigOverridePolicy)connectorClientConfigOverridePolicy);
    }
    try {
        return (Worker)CTR_WORKER_22.newInstance(workerId, time, plugins, config, offsetBackingStore);
    } catch (Throwable t) {
        throw new ConnectException(t);
    }
}
 
Example #3
Source File: GroupMetadataManagerTest.java    From kop with Apache License 2.0 6 votes vote down vote up
private ByteBuffer newMemoryRecordsBuffer(List<SimpleRecord> records,
                                          long producerId,
                                          short producerEpoch,
                                          boolean isTxnOffsetCommit) {
    TimestampType timestampType = TimestampType.CREATE_TIME;
    long timestamp = Time.SYSTEM.milliseconds();

    ByteBuffer buffer = ByteBuffer.allocate(
        AbstractRecords.estimateSizeInBytes(
            RecordBatch.CURRENT_MAGIC_VALUE, offsetConfig.offsetsTopicCompressionType(), records
        )
    );

    MemoryRecordsBuilder builder = MemoryRecords.builder(
        buffer, RecordBatch.CURRENT_MAGIC_VALUE, offsetConfig.offsetsTopicCompressionType(),
        timestampType, 0L, timestamp,
        producerId,
        producerEpoch,
        0,
        isTxnOffsetCommit,
        RecordBatch.NO_PARTITION_LEADER_EPOCH
    );
    records.forEach(builder::append);
    return builder.build().buffer();
}
 
Example #4
Source File: DelayedOperationTest.java    From kop with Apache License 2.0 6 votes vote down vote up
@Test
public void testRequestExpiry() throws Exception {
    long expiration = 20L;
    long start = Time.SYSTEM.hiResClockMs();
    MockDelayedOperation r1 = new MockDelayedOperation(expiration);
    MockDelayedOperation r2 = new MockDelayedOperation(200000L);
    assertFalse(
        "r1 not satisfied and hence watched",
        purgatory.tryCompleteElseWatch(r1, Lists.newArrayList("test1")));
    assertFalse(
        "r2 not satisfied and hence watched",
        purgatory.tryCompleteElseWatch(r2, Lists.newArrayList("test2")));
    r1.awaitExpiration();
    long elapsed = Time.SYSTEM.hiResClockMs() - start;
    assertTrue
        ("r1 completed due to expiration",
        r1.isCompleted());
    assertFalse("r2 hasn't completed", r2.isCompleted());
    assertTrue(
        "Time for expiration $elapsed should at least " + expiration,
        elapsed >= expiration);
}
 
Example #5
Source File: GroupMetadataManager.java    From kop with Apache License 2.0 6 votes vote down vote up
GroupMetadataManager(OffsetConfig offsetConfig,
                     ProducerBuilder<ByteBuffer> metadataTopicProducerBuilder,
                     ReaderBuilder<ByteBuffer> metadataTopicConsumerBuilder,
                     ScheduledExecutorService scheduler,
                     Time time,
                     Function<String, Integer> partitioner) {
    this.offsetConfig = offsetConfig;
    this.compressionType = offsetConfig.offsetsTopicCompressionType();
    this.groupMetadataCache = new ConcurrentHashMap<>();
    this.groupMetadataTopicPartitionCount = offsetConfig.offsetsTopicNumPartitions();
    this.metadataTopicProducerBuilder = metadataTopicProducerBuilder;
    this.metadataTopicReaderBuilder = metadataTopicConsumerBuilder;
    this.scheduler = scheduler;
    this.time = time;
    this.partitioner = partitioner;
}
 
Example #6
Source File: GroupMetadataManagerTest.java    From kop with Apache License 2.0 6 votes vote down vote up
private int completeTransactionalOffsetCommit(ByteBuffer buffer,
                                              long producerId,
                                              short producerEpoch,
                                              long baseOffset,
                                              boolean isCommit) {
    MemoryRecordsBuilder builder = MemoryRecords.builder(
        buffer, RecordBatch.MAGIC_VALUE_V2, CompressionType.NONE,
        TimestampType.LOG_APPEND_TIME, baseOffset, Time.SYSTEM.milliseconds(),
        producerId, producerEpoch, 0, true, true,
        RecordBatch.NO_PARTITION_LEADER_EPOCH);
    ControlRecordType controlRecordType;
    if (isCommit) {
        controlRecordType = ControlRecordType.COMMIT;
    } else {
        controlRecordType = ControlRecordType.ABORT;
    }
    builder.appendEndTxnMarker(Time.SYSTEM.milliseconds(), new EndTransactionMarker(controlRecordType, 0));
    builder.build();
    return 1;
}
 
Example #7
Source File: MultiClusterTopicManagementService.java    From kafka-monitor with Apache License 2.0 6 votes vote down vote up
void maybeElectLeader() throws Exception {
  if (!_preferredLeaderElectionRequested) {
    return;
  }

  try (KafkaZkClient zkClient = KafkaZkClient.apply(_zkConnect, JaasUtils.isZkSecurityEnabled(), com.linkedin.kmf.common.Utils.ZK_SESSION_TIMEOUT_MS,
      com.linkedin.kmf.common.Utils.ZK_CONNECTION_TIMEOUT_MS, Integer.MAX_VALUE, Time.SYSTEM, METRIC_GROUP_NAME, "SessionExpireListener", null)) {
    if (!zkClient.reassignPartitionsInProgress()) {
      List<TopicPartitionInfo> partitionInfoList = _adminClient
          .describeTopics(Collections.singleton(_topic)).all().get().get(_topic).partitions();
      LOGGER.info(
          "MultiClusterTopicManagementService will trigger requested preferred leader election for the"
              + " topic {} in cluster.", _topic);
      triggerPreferredLeaderElection(partitionInfoList, _topic);
      _preferredLeaderElectionRequested = false;
    }
  }
}
 
Example #8
Source File: KafkaTopicRepository.java    From nakadi with MIT License 6 votes vote down vote up
private KafkaZkClient createZkClient() {
    // The calling method should make sure to close connection
    return new KafkaZkClient(
            new ZooKeeperClient(
                    kafkaZookeeper.getZookeeperConnectionString(),
                    zookeeperSettings.getZkSessionTimeoutMs(),
                    zookeeperSettings.getZkConnectionTimeoutMs(),
                    zookeeperSettings.getMaxInFlightRequests(),
                    Time.SYSTEM,
                    ZookeeperSettings.METRIC_GROUP,
                    ZookeeperSettings.METRIC_TYPE
            ),
            false,
            Time.SYSTEM
    );
}
 
Example #9
Source File: NetworkClientProvider.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Creates a new network client with the given properties.
 *
 * @return A new network client with the given properties.
 */
NetworkClient createNetworkClient(long connectionMaxIdleMS,
                                  Metrics metrics,
                                  Time time,
                                  String metricGrpPrefix,
                                  ChannelBuilder channelBuilder,
                                  Metadata metadata,
                                  String clientId,
                                  int maxInFlightRequestsPerConnection,
                                  long reconnectBackoffMs,
                                  long reconnectBackoffMax,
                                  int socketSendBuffer,
                                  int socketReceiveBuffer,
                                  int defaultRequestTimeoutMs,
                                  boolean discoverBrokerVersions,
                                  ApiVersions apiVersions);
 
Example #10
Source File: WorkerBootStrap.java    From DataLink with Apache License 2.0 6 votes vote down vote up
private Keeper buildKeeper(WorkerConfig config, Worker worker, Time time, String restUrl, String bootMode) {
    Keeper keeper;
    if (BootMode.DISTRIBUTED.equals(bootMode)) {
        keeper = new WorkerKeeper(
                config,
                time,
                worker,
                new TaskStatusManager(DataLinkFactory.getObject(TaskStatusService.class)),
                new TaskConfigManager(config.getString(WorkerConfig.GROUP_ID_CONFIG), DataLinkFactory.getObject(TaskConfigService.class)),
                restUrl
        );
    } else if (BootMode.STANDALONE.equals(bootMode)) {
        keeper = new StandaloneWorkerKeeper(
                config,
                worker,
                time,
                new TaskConfigManager(config.getString(WorkerConfig.GROUP_ID_CONFIG), DataLinkFactory.getObject(TaskConfigService.class))
        );
    } else {
        throw new DatalinkException("invalid boot mode : " + bootMode);
    }

    return keeper;
}
 
Example #11
Source File: ConnectEmbedded.java    From hello-kafka-streams with Apache License 2.0 6 votes vote down vote up
public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception {
    Time time = new SystemTime();
    DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig));

    KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
    offsetBackingStore.configure(config);

    //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random
    String workerId = UUID.randomUUID().toString();
    worker = new Worker(workerId, time, config, offsetBackingStore);

    StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter());
    statusBackingStore.configure(config);

    ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter());
    configBackingStore.configure(config);

    //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything
    herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, "");
    this.connectorConfigs = connectorConfigs;

    shutdownHook = new ShutdownHook();
}
 
Example #12
Source File: ServiceKafkaClient.java    From ranger with Apache License 2.0 6 votes vote down vote up
private List<String> getTopicList(List<String> ignoreTopicList) throws Exception {
	List<String> ret = new ArrayList<String>();

	int sessionTimeout = 5000;
	int connectionTimeout = 10000;
	ZooKeeperClient zookeeperClient = new ZooKeeperClient(zookeeperConnect, sessionTimeout, connectionTimeout,
			1, Time.SYSTEM, "kafka.server", "SessionExpireListener", Option.empty());
	try (KafkaZkClient kafkaZkClient = new KafkaZkClient(zookeeperClient, true, Time.SYSTEM)) {
		Iterator<String> iter = kafkaZkClient.getAllTopicsInCluster().iterator();
		while (iter.hasNext()) {
			String topic = iter.next();
			if (ignoreTopicList == null || !ignoreTopicList.contains(topic)) {
				ret.add(topic);
			}
		}
	}
	return ret;
}
 
Example #13
Source File: MiniKafkaCluster.java    From incubator-pinot with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"rawtypes", "unchecked"})
private MiniKafkaCluster(List<String> brokerIds)
    throws IOException, InterruptedException {
  this.zkServer = new EmbeddedZooKeeper();
  this.tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "mini-kafka-cluster");
  this.kafkaServer = new ArrayList<>();
  int port = 0;
  for (String id : brokerIds) {
    port = getAvailablePort();
    KafkaConfig c = new KafkaConfig(createBrokerConfig(id, port));
    Seq seq =
        scala.collection.JavaConverters.collectionAsScalaIterableConverter(Collections.emptyList()).asScala().toSeq();
    kafkaServer.add(new KafkaServer(c, Time.SYSTEM, Option.empty(), seq));
  }
  Properties props = new Properties();
  props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "127.0.0.1:" + port);
  adminClient = AdminClient.create(props);
}
 
Example #14
Source File: KafkaCruiseControl.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Package private constructor for unit tests w/o static state initialization.
 */
KafkaCruiseControl(KafkaCruiseControlConfig config,
                   Time time,
                   AnomalyDetector anomalyDetector,
                   Executor executor,
                   LoadMonitor loadMonitor,
                   ExecutorService goalOptimizerExecutor,
                   GoalOptimizer goalOptimizer) {
  _config = config;
  _time = time;
  _anomalyDetector = anomalyDetector;
  _executor = executor;
  _loadMonitor = loadMonitor;
  _goalOptimizerExecutor = goalOptimizerExecutor;
  _goalOptimizer = goalOptimizer;
}
 
Example #15
Source File: ExecutionTaskManager.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * The constructor of The Execution task manager.
 *
 * @param adminClient The adminClient use to query logdir information of replicas.
 * @param dropwizardMetricRegistry The metric registry.
 * @param time The time object to get the time.
 * @param config config object that holds all Kafka Cruise control related configs
 */
public ExecutionTaskManager(AdminClient adminClient,
                            MetricRegistry dropwizardMetricRegistry,
                            Time time,
                            KafkaCruiseControlConfig config) {
  _inProgressInterBrokerReplicaMovementsByBrokerId = new HashMap<>();
  _inProgressIntraBrokerReplicaMovementsByBrokerId = new HashMap<>();
  _inProgressPartitionsForInterBrokerMovement = new HashSet<>();
  _executionTaskTracker = new ExecutionTaskTracker(dropwizardMetricRegistry, time);
  _executionTaskPlanner = new ExecutionTaskPlanner(adminClient, config);
  _defaultInterBrokerPartitionMovementConcurrency = config.getInt(ExecutorConfig.NUM_CONCURRENT_PARTITION_MOVEMENTS_PER_BROKER_CONFIG);
  _defaultIntraBrokerPartitionMovementConcurrency = config.getInt(ExecutorConfig.NUM_CONCURRENT_INTRA_BROKER_PARTITION_MOVEMENTS_CONFIG);
  _defaultLeadershipMovementConcurrency = config.getInt(ExecutorConfig.NUM_CONCURRENT_LEADER_MOVEMENTS_CONFIG);
  _maxNumClusterMovementConcurrency = config.getInt(ExecutorConfig.MAX_NUM_CLUSTER_MOVEMENTS_CONFIG);
  _brokersToSkipConcurrencyCheck = new HashSet<>();
  _isKafkaAssignerMode = false;
  _requestedInterBrokerPartitionMovementConcurrency = null;
  _requestedIntraBrokerPartitionMovementConcurrency = null;
  _requestedLeadershipMovementConcurrency = null;
}
 
Example #16
Source File: KafkaNetworkClientProvider.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Override
public NetworkClient createNetworkClient(long connectionMaxIdleMS,
                                         Metrics metrics,
                                         Time time,
                                         String metricGrpPrefix,
                                         ChannelBuilder channelBuilder,
                                         Metadata metadata,
                                         String clientId,
                                         int maxInFlightRequestsPerConnection,
                                         long reconnectBackoffMs,
                                         long reconnectBackoffMax,
                                         int socketSendBuffer,
                                         int socketReceiveBuffer,
                                         int defaultRequestTimeoutMs,
                                         boolean discoverBrokerVersions,
                                         ApiVersions apiVersions) {
  return new NetworkClient(new Selector(connectionMaxIdleMS, metrics, time, metricGrpPrefix, channelBuilder, new LogContext()),
                           metadata, clientId, maxInFlightRequestsPerConnection, reconnectBackoffMs,
                           reconnectBackoffMax, socketSendBuffer, socketReceiveBuffer, defaultRequestTimeoutMs,
                           ClientDnsLookup.DEFAULT, time, discoverBrokerVersions, apiVersions, new LogContext());
}
 
Example #17
Source File: KafkaStore.java    From data-highway with Apache License 2.0 6 votes vote down vote up
public KafkaStore(
    String bootstrapServers,
    Serializer<K, V> serializer,
    String topic,
    Collection<StoreUpdateObserver<K, V>> observers,
    Time time,
    Map<String, Object> additionalProducerProps,
    Map<String, Object> additionalConsumerProps) {
  this.serializer = serializer;
  this.topic = topic;
  this.observers = new ArrayList<>(observers);

  localStore = new HashMap<>();

  kafkaLog = createKafkaLog(bootstrapServers, topic, time, additionalProducerProps, additionalConsumerProps);

  kafkaLog.start();
}
 
Example #18
Source File: BrokerFailureDetectorTest.java    From cruise-control with BSD 2-Clause "Simplified" License 6 votes vote down vote up
@Test
public void testLoadFailedBrokersFromZK() throws Exception {
  Time mockTime = getMockTime();
  Queue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator());
  BrokerFailureDetector detector = createBrokerFailureDetector(anomalies, mockTime);

  try {
    detector.startDetection();
    int brokerId = 0;
    killBroker(brokerId);
    long start = System.currentTimeMillis();
    while (anomalies.isEmpty() && System.currentTimeMillis() < start + 30000) {
      // Wait for the anomalies to be drained.
    }
    assertEquals(Collections.singletonMap(brokerId, 100L), detector.failedBrokers());
    // shutdown, advance the clock and create a new detector.
    detector.shutdown();
    mockTime.sleep(100L);
    detector = createBrokerFailureDetector(anomalies, mockTime);
    // start the newly created detector and the broker down time should remain previous time.
    detector.startDetection();
    assertEquals(Collections.singletonMap(brokerId, 100L), detector.failedBrokers());
  } finally {
    detector.shutdown();
  }
}
 
Example #19
Source File: MinimumFileAgePredicateTest.java    From kafka-connect-spooldir with Apache License 2.0 5 votes vote down vote up
@Test
public void oldEnough() throws IOException {
  long timestamp = 1559653835123L;
  this.inputFile.setLastModified(timestamp);
  timestamp += 5000L;
  Time time = time(timestamp);
  InputFileDequeue.MinimumFileAgePredicate predicate = new InputFileDequeue.MinimumFileAgePredicate(
      1000,
      time
  );
  assertTrue(predicate.test(this.inputFile), "File should be old enough");
}
 
Example #20
Source File: BootstrapTask.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
BootstrapTask(long startMs,
              long endMs,
              boolean clearMetrics,
              MetadataClient metadataClient,
              KafkaPartitionMetricSampleAggregator metricSampleAggregator,
              LoadMonitorTaskRunner loadMonitorTaskRunner,
              MetricFetcherManager metricFetcherManager,
              SampleStore sampleStore,
              int configuredNumSnapshots,
              long configuredSnapshotWindowMs,
              long samplingIntervalMs,
              Time time) {
  if (startMs < 0 || endMs < 0 || endMs <= startMs) {
    throw new IllegalArgumentException(String.format("Invalid bootstrap time range [%d, %d]. The bootstrap end "
                                                         + "time must be non negative and the end time "
                                                         + "must be greater than start time.", startMs, endMs));
  }
  _mode = BootstrapMode.RANGE;
  _startMs = startMs;
  _endMs = endMs;
  _clearMetrics = clearMetrics;
  _metadataClient = metadataClient;
  _metricSampleAggregator = metricSampleAggregator;
  _loadMonitorTaskRunner = loadMonitorTaskRunner;
  _metricFetcherManager = metricFetcherManager;
  _sampleStore = sampleStore;
  _configuredNumSnapshots = configuredNumSnapshots;
  _configuredSnapshotWindowMs = configuredSnapshotWindowMs;
  _samplingIntervalMs = samplingIntervalMs;
  _time = time;
  _bootstrappedRangeStartMs = startMs;
  _bootstrappedRangeEndMs = startMs;
}
 
Example #21
Source File: TimeSerializationModuleTest.java    From connect-utils with Apache License 2.0 5 votes vote down vote up
@Test
public void roundtrip() throws IOException {
  final Time expected = mock(Time.class);
  when(expected.milliseconds()).thenReturn(1485910473123L);
  when(expected.nanoseconds()).thenReturn(1485910473123123L);
  final String temp = ObjectMapperFactory.INSTANCE.writeValueAsString(expected);
  log.trace(temp);
  final Time actual = ObjectMapperFactory.INSTANCE.readValue(temp, Time.class);
  assertNotNull(actual);
  assertEquals(expected.milliseconds(), actual.milliseconds(), "milliseconds() does not match.");
  assertEquals(expected.nanoseconds(), actual.nanoseconds(), "nanoseconds() does not match.");
}
 
Example #22
Source File: KafkaEventChannel.java    From jstarcraft-core with Apache License 2.0 5 votes vote down vote up
@Override
public void registerMonitor(Set<Class> types, EventMonitor monitor) {
    try {
        for (Class type : types) {
            EventManager manager = managers.get(type);
            String group = name + StringUtility.DOT + type.getName();
            if (manager == null) {
                manager = new EventManager();
                managers.put(type, manager);
                Properties properties = new Properties();
                properties.put("bootstrap.servers", connections);
                properties.put("key.deserializer", keyDeserializer);
                properties.put("value.deserializer", valueDeserializer);
                switch (mode) {
                case QUEUE: {
                    properties.put("group.id", group);
                    properties.put("auto.offset.reset", "earliest");
                    break;
                }
                case TOPIC: {
                    properties.put("group.id", group + UUID.randomUUID());
                    properties.put("auto.offset.reset", "latest");
                    break;
                }
                }
                KafkaConsumer<String, byte[]> consumer = new KafkaConsumer<>(properties);
                consumer.subscribe(Collections.singleton(group));
                // TODO 此处是为了防止auto.offset.reset为latest时,可能会丢失第一次poll之前的消息.
                updateAssignmentMetadata.invoke(consumer, Time.SYSTEM.timer(Long.MAX_VALUE));
                consumers.put(type, consumer);
                EventThread thread = new EventThread(type, manager);
                thread.start();
                threads.put(type, thread);
            }
            manager.attachMonitor(monitor);
        }
    } catch (Exception exception) {
        throw new RuntimeException(exception);
    }
}
 
Example #23
Source File: SourceRecordDequeImpl.java    From connect-utils with Apache License 2.0 5 votes vote down vote up
SourceRecordDequeImpl(Time time,
                      int maximumCapacity,
                      int batchSize,
                      int emptyWaitMs, int maximumCapacityWaitMs, int maximumCapacityTimeoutMs, RateLimiter writeRateLimit) {
  super();
  this.time = time;
  this.batchSize = batchSize;
  this.maximumCapacity = maximumCapacity;
  this.emptyWaitMs = emptyWaitMs;
  this.maximumCapacityWaitMs = maximumCapacityWaitMs;
  this.maximumCapacityTimeoutMs = maximumCapacityTimeoutMs;
  this.writeRateLimit = writeRateLimit;
}
 
Example #24
Source File: KafkaStarterUtils.java    From uReplicator with Apache License 2.0 5 votes vote down vote up
public static void createTopic(String kafkaTopic, int numOfPartitions, String zkStr, String replicatorFactor) {
  // TopicCommand.main() will call System.exit() finally, which will break maven-surefire-plugin
  try {
    String[] args = new String[]{"--create", "--zookeeper", zkStr, "--replication-factor", replicatorFactor,
        "--partitions", String.valueOf(numOfPartitions), "--topic", kafkaTopic};
    KafkaZkClient zkClient = KafkaZkClient
        .apply(zkStr, false, 3000, 3000, Integer.MAX_VALUE, Time.SYSTEM, "kafka.server",
            "SessionExpireListener");
    TopicCommand.TopicCommandOptions opts = new TopicCommand.TopicCommandOptions(args);
    TopicCommand.createTopic(zkClient, opts);
  } catch (TopicExistsException e) {
    // Catch TopicExistsException otherwise it will break maven-surefire-plugin
    System.out.println("Topic already existed");
  }
}
 
Example #25
Source File: KafkaNodeClient.java    From feeyo-redisproxy with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public ClientResponse sendAndRecvice(ClientRequest clientRequest) throws IOException {
	
	//
	NetworkClientUtils.awaitReady(client, node, Time.SYSTEM, 5000);
	
	//
	ClientResponse response = NetworkClientUtils.sendAndReceive(client, clientRequest, Time.SYSTEM);
	return response;
}
 
Example #26
Source File: ConnectRunner.java    From aiven-kafka-connect-gcs with GNU Affero General Public License v3.0 5 votes vote down vote up
void start() {
    final Map<String, String> workerProps = new HashMap<>();
    workerProps.put("bootstrap.servers", bootstrapServers);

    workerProps.put("offset.flush.interval.ms", Integer.toString(offsetFlushInterval));

    // These don't matter much (each connector sets its own converters), but need to be filled with valid classes.
    workerProps.put("key.converter", "org.apache.kafka.connect.converters.ByteArrayConverter");
    workerProps.put("value.converter", "org.apache.kafka.connect.converters.ByteArrayConverter");
    workerProps.put("internal.key.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.key.converter.schemas.enable", "false");
    workerProps.put("internal.value.converter", "org.apache.kafka.connect.json.JsonConverter");
    workerProps.put("internal.value.converter.schemas.enable", "false");

    // Don't need it since we'll memory MemoryOffsetBackingStore.
    workerProps.put("offset.storage.file.filename", "");

    workerProps.put("plugin.path", pluginDir.getPath());

    final Time time = Time.SYSTEM;
    final String workerId = "test-worker";

    final Plugins plugins = new Plugins(workerProps);
    final StandaloneConfig config = new StandaloneConfig(workerProps);

    final Worker worker = new Worker(
        workerId, time, plugins, config, new MemoryOffsetBackingStore());
    herder = new StandaloneHerder(worker);

    final RestServer rest = new RestServer(config);

    connect = new Connect(herder, rest);

    connect.start();
}
 
Example #27
Source File: StandaloneWorkerKeeper.java    From DataLink with Apache License 2.0 5 votes vote down vote up
public StandaloneWorkerKeeper(WorkerConfig config,
                              Worker worker,
                              Time time,
                              TaskConfigManager taskConfigManager
) {
    this.workerConfig = config;
    this.workerId = worker.workerId();
    this.worker = worker;
    this.time = time;
    this.taskConfigManager = taskConfigManager;
    this.stopping = new AtomicBoolean(false);
    this.configState = ClusterConfigState.EMPTY;
    this.forwardRequestExecutor = Executors.newSingleThreadExecutor();
    taskConfigManager.setUpdateListener(new ConfigUpdateListener());
}
 
Example #28
Source File: Worker.java    From DataLink with Apache License 2.0 5 votes vote down vote up
public Worker(String workerId, Time time, WorkerConfig config, TaskPositionManager taskPositionManager, TaskSyncStatusManager taskSyncStatusManager,
              ProbeManager probeManager) {
    this.executor = Executors.newCachedThreadPool(new NamedThreadFactory("Task-Container"));
    this.workerId = workerId;
    this.time = time;
    this.config = config;
    this.taskPositionManager = taskPositionManager;
    this.taskSyncStatusManager = taskSyncStatusManager;
    this.probeManager = probeManager;
}
 
Example #29
Source File: ConnectStandalone.java    From mongo-kafka with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
ConnectStandalone(final Properties workerProperties) {
  Time time = Time.SYSTEM;
  LOGGER.info("Kafka Connect standalone worker initializing ...");
  long initStart = time.hiResClockMs();
  WorkerInfo initInfo = new WorkerInfo();
  initInfo.logAll();

  Map<String, String> workerProps = (Map) workerProperties;

  LOGGER.info("Scanning for plugin classes. This might take a moment ...");
  Plugins plugins = new Plugins(workerProps);
  plugins.compareAndSwapWithDelegatingLoader();
  StandaloneConfig config = new StandaloneConfig(workerProps);

  String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
  LOGGER.debug("Kafka cluster ID: {}", kafkaClusterId);

  RestServer rest = new RestServer(config);
  URI advertisedUrl = rest.advertisedUrl();
  String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();

  Worker worker = new Worker(workerId, time, plugins, config, new FileOffsetBackingStore());
  this.herder = new StandaloneHerder(worker, kafkaClusterId);
  connectionString = advertisedUrl.toString() + herder.kafkaClusterId();

  this.connect = new Connect(herder, rest);
  LOGGER.info(
      "Kafka Connect standalone worker initialization took {}ms",
      time.hiResClockMs() - initStart);
}
 
Example #30
Source File: BootstrapTask.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
BootstrapTask(boolean clearMetrics,
              MetadataClient metadataClient,
              KafkaPartitionMetricSampleAggregator metricSampleAggregator,
              LoadMonitorTaskRunner loadMonitorTaskRunner,
              MetricFetcherManager metricFetcherManager,
              SampleStore sampleStore,
              int configuredNumSnapshots,
              long configuredSnapshotWindowMs,
              long samplingIntervalMs,
              Time time) {
  _mode = BootstrapMode.RECENT;
  _startMs = -1L;
  _endMs = -1L;
  _clearMetrics = clearMetrics;
  _metadataClient = metadataClient;
  _metricSampleAggregator = metricSampleAggregator;
  _loadMonitorTaskRunner = loadMonitorTaskRunner;
  _metricFetcherManager = metricFetcherManager;
  _sampleStore = sampleStore;
  _configuredNumSnapshots = configuredNumSnapshots;
  _configuredSnapshotWindowMs = configuredSnapshotWindowMs;
  _samplingIntervalMs = samplingIntervalMs;
  _time = time;
  long now = _time.milliseconds();
  _bootstrappedRangeStartMs = now;
  _bootstrappedRangeEndMs = now;
}