org.apache.kafka.connect.util.ConnectorUtils Java Examples

The following examples show how to use org.apache.kafka.connect.util.ConnectorUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaSourceConnector.java    From MirrorTool-for-Kafka-Connect with Apache License 2.0 6 votes vote down vote up
@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
  List<String> leaderTopicPartitions = partitionMonitor.getCurrentLeaderTopicPartitions().stream()
      .map(LeaderTopicPartition::toString).sorted() // Potential task performance/overhead improvement by roughly
                                                    // grouping tasks and leaders
      .collect(Collectors.toList());
  int taskCount = Math.min(maxTasks, leaderTopicPartitions.size());
  if (taskCount < 1) {
    logger.warn("No tasks to start.");
    return new ArrayList<>();
  }
  return ConnectorUtils.groupPartitions(leaderTopicPartitions, taskCount).stream().map(leaderTopicPartitionsGroup -> {
    Map<String, String> taskConfig = new HashMap<>();
    taskConfig.putAll(connectorConfig.allAsStrings());
    taskConfig.put(KafkaSourceConnectorConfig.TASK_LEADER_TOPIC_PARTITION_CONFIG,
        String.join(",", leaderTopicPartitionsGroup));
    return taskConfig;
  }).collect(Collectors.toList());
}
 
Example #2
Source File: MongodbSourceConnector.java    From kafka-connect-mongodb with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a set of configuration for the Task based on the current configuration.
 *
 * @param maxTasks maximum number of configurations to generate
 * @return configurations for the Task
 */
@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
    ArrayList<Map<String, String>> configs = new ArrayList<>();
    List<String> dbs = Arrays.asList(databases.split(","));
    int numGroups = Math.min(dbs.size(), maxTasks);
    List<List<String>> dbsGrouped = ConnectorUtils.groupPartitions(dbs, numGroups);
    for (int i = 0; i < numGroups; i++) {
        Map<String, String> config = new HashMap<>();
        config.put(URI, uri);
        if(host!=null){
        	config.put(HOST, host);
        }
        if(port!=null){
        	config.put(PORT, port);
        }
        config.put(SCHEMA_NAME, schemaName);
        config.put(BATCH_SIZE, batchSize);
        config.put(TOPIC_PREFIX, topicPrefix);
        config.put(CONVERTER_CLASS, converterClass);
        config.put(DATABASES, StringUtils.join(dbsGrouped.get(i), ","));
        configs.add(config);
    }
    return configs;
}
 
Example #3
Source File: ZeebeSourceConnector.java    From kafka-connect-zeebe with Apache License 2.0 5 votes vote down vote up
/** {@inheritDoc} */
@Override
public List<Map<String, String>> taskConfigs(final int maxTasks) {
  return ConnectorUtils.groupPartitions(jobTypes, maxTasks)
      .stream()
      .map(this::transformTask)
      .collect(Collectors.toList());
}
 
Example #4
Source File: MongodbSinkConnector.java    From kafka-connect-mongodb with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a set of configurations for Tasks based on the current configuration,
 * producing at most maxTasks configurations.
 *
 * @param maxTasks maximum number of task to start
 * @return configurations for tasks
 */
@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
    List<Map<String, String>> configs = new ArrayList<>();
    List<String> coll = Arrays.asList(collections.split(","));
    int numGroups = Math.min(coll.size(), maxTasks);
    List<List<String>> dbsGrouped = ConnectorUtils.groupPartitions(coll, numGroups);
    List<String> topics = Arrays.asList(this.topics.split(","));
    List<List<String>> topicsGrouped = ConnectorUtils.groupPartitions(topics, numGroups);

    for (int i = 0; i < numGroups; i++) {
        Map<String, String> config = new HashMap<>();
        config.put(URI, uri);
        if(host!=null){
            config.put(HOST, host);
        }
        if(port!=null){
        	config.put(PORT, port);
        }
        config.put(BULK_SIZE, bulkSize);
        config.put(DATABASE, database);
        config.put(COLLECTIONS, StringUtils.join(dbsGrouped.get(i), ","));
        config.put(TOPICS, StringUtils.join(topicsGrouped.get(i), ","));
        configs.add(config);
    }
    return configs;
}
 
Example #5
Source File: DynamoDbSourceConnector.java    From kafka-connect-dynamodb with Apache License 2.0 5 votes vote down vote up
@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
    return ConnectorUtils.groupPartitions(new ArrayList<>(streamShards.keySet()), maxTasks).stream().map(taskShards -> {
        final Map<String, String> taskConfig = new HashMap<>();
        taskConfig.put(TaskConfig.Keys.REGION, config.region.getName());
        taskConfig.put(TaskConfig.Keys.TOPIC_FORMAT, config.topicFormat);
        taskConfig.put(TaskConfig.Keys.SHARDS, taskShards.stream().map(Shard::getShardId).collect(Collectors.joining(",")));
        taskShards.forEach(shard -> {
            final TableDescription tableDesc = streamShards.get(shard);
            taskConfig.put(shard.getShardId() + "." + TaskConfig.Keys.TABLE, tableDesc.getTableName());
            taskConfig.put(shard.getShardId() + "." + TaskConfig.Keys.STREAM_ARN, tableDesc.getLatestStreamArn());
        });
        return taskConfig;
    }).collect(Collectors.toList());
}
 
Example #6
Source File: TaskConfigBuilder.java    From kafka-connect-jenkins with Apache License 2.0 5 votes vote down vote up
/**
 * Builds task config for each task.
 *
 * @param work The entire work expected to be fulfilled by this connector
 * @return The task configuration for each of the tasks
 */
public List<Map<String, String>> build(List<T> work) {
    //Create job groups
    int workSizeCount = work.size();
    int numGroups = Math.min(workSizeCount, maxTasks);

    logger.debug("Total work size: {} units. maxTasks: {}. numGroups: {}.", workSizeCount, maxTasks, numGroups);

    List<List<T>> workGroups = ConnectorUtils.groupPartitions(work, numGroups);
    logger.debug("Number of work groups created: {}.", workGroups.size());

    //Create task configs for each group
    List<Map<String, String>> taskConfigs = new ArrayList<>(workGroups.size());

    for (List<T> group : workGroups) {
        //Forward the config from connector to SourceTask so that they have access to the configured TOPIC NAME
        Map<String, String> taskProps = new HashMap<>(connectorCfg.originalsStrings());

        //Concatenate all job urls that need to be handled by a single task
        String commaSeparatedJobUrls = group.stream()
                .map(j -> taskCfgExtractor.extract(j))
                .collect(Collectors.joining(","));

        taskProps.put(taskCfgKey, commaSeparatedJobUrls);
        logger.trace("taskProps: {}", taskProps);
        taskConfigs.add(taskProps);
    }
    return taskConfigs;
}
 
Example #7
Source File: RangeTaskAssignor.java    From mirus with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
@Override
public List<List<TopicPartition>> assign(List<TopicPartition> partitions, int numGroups) {
  return ConnectorUtils.groupPartitions(partitions, numGroups);
}