org.apache.kafka.connect.util.ConnectorUtils Java Examples

@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
  List<String> leaderTopicPartitions = partitionMonitor.getCurrentLeaderTopicPartitions().stream()
      .map(LeaderTopicPartition::toString).sorted() // Potential task performance/overhead improvement by roughly
                                                    // grouping tasks and leaders
      .collect(Collectors.toList());
  int taskCount = Math.min(maxTasks, leaderTopicPartitions.size());
  if (taskCount < 1) {
    logger.warn("No tasks to start.");
    return new ArrayList<>();
  }
  return ConnectorUtils.groupPartitions(leaderTopicPartitions, taskCount).stream().map(leaderTopicPartitionsGroup -> {
    Map<String, String> taskConfig = new HashMap<>();
    taskConfig.putAll(connectorConfig.allAsStrings());
    taskConfig.put(KafkaSourceConnectorConfig.TASK_LEADER_TOPIC_PARTITION_CONFIG,
        String.join(",", leaderTopicPartitionsGroup));
    return taskConfig;
  }).collect(Collectors.toList());
}
 

/**
 * Returns a set of configuration for the Task based on the current configuration.
 *
 * @param maxTasks maximum number of configurations to generate
 * @return configurations for the Task
 */
@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
    ArrayList<Map<String, String>> configs = new ArrayList<>();
    List<String> dbs = Arrays.asList(databases.split(","));
    int numGroups = Math.min(dbs.size(), maxTasks);
    List<List<String>> dbsGrouped = ConnectorUtils.groupPartitions(dbs, numGroups);
    for (int i = 0; i < numGroups; i++) {
        Map<String, String> config = new HashMap<>();
        config.put(URI, uri);
        if(host!=null){
        	config.put(HOST, host);
        }
        if(port!=null){
        	config.put(PORT, port);
        }
        config.put(SCHEMA_NAME, schemaName);
        config.put(BATCH_SIZE, batchSize);
        config.put(TOPIC_PREFIX, topicPrefix);
        config.put(CONVERTER_CLASS, converterClass);
        config.put(DATABASES, StringUtils.join(dbsGrouped.get(i), ","));
        configs.add(config);
    }
    return configs;
}
 

/** {@inheritDoc} */
@Override
public List<Map<String, String>> taskConfigs(final int maxTasks) {
  return ConnectorUtils.groupPartitions(jobTypes, maxTasks)
      .stream()
      .map(this::transformTask)
      .collect(Collectors.toList());
}
 

/**
 * Returns a set of configurations for Tasks based on the current configuration,
 * producing at most maxTasks configurations.
 *
 * @param maxTasks maximum number of task to start
 * @return configurations for tasks
 */
@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
    List<Map<String, String>> configs = new ArrayList<>();
    List<String> coll = Arrays.asList(collections.split(","));
    int numGroups = Math.min(coll.size(), maxTasks);
    List<List<String>> dbsGrouped = ConnectorUtils.groupPartitions(coll, numGroups);
    List<String> topics = Arrays.asList(this.topics.split(","));
    List<List<String>> topicsGrouped = ConnectorUtils.groupPartitions(topics, numGroups);

    for (int i = 0; i < numGroups; i++) {
        Map<String, String> config = new HashMap<>();
        config.put(URI, uri);
        if(host!=null){
            config.put(HOST, host);
        }
        if(port!=null){
        	config.put(PORT, port);
        }
        config.put(BULK_SIZE, bulkSize);
        config.put(DATABASE, database);
        config.put(COLLECTIONS, StringUtils.join(dbsGrouped.get(i), ","));
        config.put(TOPICS, StringUtils.join(topicsGrouped.get(i), ","));
        configs.add(config);
    }
    return configs;
}
 

@Override
public List<Map<String, String>> taskConfigs(int maxTasks) {
    return ConnectorUtils.groupPartitions(new ArrayList<>(streamShards.keySet()), maxTasks).stream().map(taskShards -> {
        final Map<String, String> taskConfig = new HashMap<>();
        taskConfig.put(TaskConfig.Keys.REGION, config.region.getName());
        taskConfig.put(TaskConfig.Keys.TOPIC_FORMAT, config.topicFormat);
        taskConfig.put(TaskConfig.Keys.SHARDS, taskShards.stream().map(Shard::getShardId).collect(Collectors.joining(",")));
        taskShards.forEach(shard -> {
            final TableDescription tableDesc = streamShards.get(shard);
            taskConfig.put(shard.getShardId() + "." + TaskConfig.Keys.TABLE, tableDesc.getTableName());
            taskConfig.put(shard.getShardId() + "." + TaskConfig.Keys.STREAM_ARN, tableDesc.getLatestStreamArn());
        });
        return taskConfig;
    }).collect(Collectors.toList());
}
 

/**
 * Builds task config for each task.
 *
 * @param work The entire work expected to be fulfilled by this connector
 * @return The task configuration for each of the tasks
 */
public List<Map<String, String>> build(List<T> work) {
    //Create job groups
    int workSizeCount = work.size();
    int numGroups = Math.min(workSizeCount, maxTasks);

    logger.debug("Total work size: {} units. maxTasks: {}. numGroups: {}.", workSizeCount, maxTasks, numGroups);

    List<List<T>> workGroups = ConnectorUtils.groupPartitions(work, numGroups);
    logger.debug("Number of work groups created: {}.", workGroups.size());

    //Create task configs for each group
    List<Map<String, String>> taskConfigs = new ArrayList<>(workGroups.size());

    for (List<T> group : workGroups) {
        //Forward the config from connector to SourceTask so that they have access to the configured TOPIC NAME
        Map<String, String> taskProps = new HashMap<>(connectorCfg.originalsStrings());

        //Concatenate all job urls that need to be handled by a single task
        String commaSeparatedJobUrls = group.stream()
                .map(j -> taskCfgExtractor.extract(j))
                .collect(Collectors.joining(","));

        taskProps.put(taskCfgKey, commaSeparatedJobUrls);
        logger.trace("taskProps: {}", taskProps);
        taskConfigs.add(taskProps);
    }
    return taskConfigs;
}
 

@Override
public List<List<TopicPartition>> assign(List<TopicPartition> partitions, int numGroups) {
  return ConnectorUtils.groupPartitions(partitions, numGroups);
}