kafka.admin.RackAwareMode Java Examples

The following examples show how to use kafka.admin.RackAwareMode. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KafkaUnit.java    From SkaETL with Apache License 2.0 7 votes vote down vote up
public void createTopic(String topicName, int numPartitions) {
    // setup

    String zookeeperHost = zookeeperString;
    Boolean isSucre = false;
    int sessionTimeoutMs = 200000;
    int connectionTimeoutMs = 15000;
    int maxInFlightRequests = 10;
    Time time = Time.SYSTEM;
    String metricGroup = "myGroup";
    String metricType = "myType";
    KafkaZkClient zkClient = KafkaZkClient.apply(zookeeperHost,isSucre,sessionTimeoutMs,
            connectionTimeoutMs,maxInFlightRequests,time,metricGroup,metricType);
    AdminZkClient adminZkClient = new AdminZkClient(zkClient);
    try {
        // run
        LOGGER.info("Executing: CreateTopic " + topicName);
        adminZkClient.createTopic(topicName,numPartitions, 1,new Properties(), RackAwareMode.Disabled$.MODULE$);
    } finally {
        zkClient.close();
    }

}
 
Example #2
Source File: KafkaTransportProviderAdmin.java    From brooklin with BSD 2-Clause "Simplified" License 6 votes vote down vote up
/**
 * Create Kafka topic based on the destination connection string, if it does not already exist.
 * @param connectionString connection string from which to obtain topic name
 * @param numberOfPartitions number of partitions
 * @param topicConfig topic config to use for topic creation
 */
public void createTopic(String connectionString, int numberOfPartitions, Properties topicConfig) {
  Validate.notNull(connectionString, "destination should not be null");
  Validate.notNull(topicConfig, "topicConfig should not be null");
  Validate.isTrue(_zkUtils.isPresent(), "zkUtils should be present");

  String topicName = KafkaTransportProviderUtils.getTopicName(connectionString);
  populateTopicConfig(topicConfig);
  try {
    // Create only if it doesn't exist.
    if (!AdminUtils.topicExists(_zkUtils.get(), topicName)) {
      int replicationFactor = Integer.parseInt(topicConfig.getProperty("replicationFactor", DEFAULT_REPLICATION_FACTOR));
      LOG.info("Creating topic with name {} partitions={} with properties {}", topicName, numberOfPartitions,
              topicConfig);

      AdminUtils.createTopic(_zkUtils.get(), topicName, numberOfPartitions, replicationFactor, topicConfig, RackAwareMode.Disabled$.MODULE$);
    } else {
      LOG.warn("Topic with name {} already exists", topicName);
    }
  } catch (Throwable e) {
    LOG.error("Creating topic {} failed with exception {}", topicName, e);
    throw e;
  }
}
 
Example #3
Source File: KafkaRpcTopicService.java    From devicehive-java-server with Apache License 2.0 6 votes vote down vote up
public void createTopic(String topic) {
    ZkClient zkClient = new ZkClient(
            kafkaRpcConfig.getZookeeperConnect(),
            kafkaRpcConfig.getSessionTimeout(),
            kafkaRpcConfig.getConnectionTimeout(),
            ZKStringSerializer$.MODULE$);
    try {
        ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(kafkaRpcConfig.getZookeeperConnect()), false);
        Properties topicConfig = kafkaRpcConfig.topicProps();
        if (!AdminUtils.topicExists(zkUtils, topic)) {
            AdminUtils.createTopic(zkUtils, topic, kafkaRpcConfig.getNumPartitions(), 
                    kafkaRpcConfig.getReplicationFactor(), topicConfig, RackAwareMode.Enforced$.MODULE$);
        }
    } finally {
        zkClient.close();
    }
}
 
Example #4
Source File: KafkaUtils.java    From oryx with Apache License 2.0 6 votes vote down vote up
/**
 * @param zkServers Zookeeper server string: host1:port1[,host2:port2,...]
 * @param topic topic to create (if not already existing)
 * @param partitions number of topic partitions
 * @param topicProperties optional topic config properties
 */
public static void maybeCreateTopic(String zkServers,
                                    String topic,
                                    int partitions,
                                    Properties topicProperties) {
  ZkUtils zkUtils = ZkUtils.apply(zkServers, ZK_TIMEOUT_MSEC, ZK_TIMEOUT_MSEC, false);
  try {
    if (AdminUtils.topicExists(zkUtils, topic)) {
      log.info("No need to create topic {} as it already exists", topic);
    } else {
      log.info("Creating topic {} with {} partition(s)", topic, partitions);
      try {
        AdminUtils.createTopic(
            zkUtils, topic, partitions, 1, topicProperties, RackAwareMode.Enforced$.MODULE$);
        log.info("Created topic {}", topic);
      } catch (TopicExistsException re) {
        log.info("Topic {} already exists", topic);
      }
    }
  } finally {
    zkUtils.close();
  }
}
 
Example #5
Source File: KafkaTopics.java    From rya with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a set of Kafka topics for each topic that does not already exist.
 *
 * @param zookeeperServers - The Zookeeper servers that are used by the Kafka Streams program. (not null)
 * @param topicNames - The topics that will be created. (not null)
 * @param partitions - The number of partitions that each of the topics will have.
 * @param replicationFactor - The replication factor of the topics that are created.
 * @param topicProperties - The optional properties of the topics to create.
 */
public static void createTopics(
        final String zookeeperServers,
        final Set<String> topicNames,
        final int partitions,
        final int replicationFactor,
        final Optional<Properties> topicProperties) {
    requireNonNull(zookeeperServers);
    requireNonNull(topicNames);

    ZkUtils zkUtils = null;
    try {
        zkUtils = ZkUtils.apply(new ZkClient(zookeeperServers, 30000, 30000, ZKStringSerializer$.MODULE$), false);
        for(final String topicName : topicNames) {
            if(!AdminUtils.topicExists(zkUtils, topicName)) {
                AdminUtils.createTopic(zkUtils, topicName, partitions, replicationFactor, topicProperties.orElse(new Properties()), RackAwareMode.Disabled$.MODULE$);
            }
        }
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 
Example #6
Source File: KafkaDestinationProcessorTest.java    From incubator-samoa with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topic
    AdminUtils.createTopic(zkUtils, TOPIC, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example #7
Source File: KafkaUtilsTest.java    From incubator-samoa with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafkaUtils-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics
    AdminUtils.createTopic(zkUtils, TOPIC_R, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    AdminUtils.createTopic(zkUtils, TOPIC_S, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example #8
Source File: KafkaEntranceProcessorTest.java    From incubator-samoa with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpClass() throws IOException {
    // setup Zookeeper
    zkServer = new EmbeddedZookeeper();
    zkConnect = ZKHOST + ":" + zkServer.port();
    zkClient = new ZkClient(zkConnect, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkUtils zkUtils = ZkUtils.apply(zkClient, false);

    // setup Broker
    Properties brokerProps = new Properties();
    brokerProps.setProperty("zookeeper.connect", zkConnect);
    brokerProps.setProperty("broker.id", "0");
    brokerProps.setProperty("log.dirs", Files.createTempDirectory("kafka-").toAbsolutePath().toString());
    brokerProps.setProperty("listeners", "PLAINTEXT://" + BROKERHOST + ":" + BROKERPORT);
    KafkaConfig config = new KafkaConfig(brokerProps);
    Time mock = new MockTime();
    kafkaServer = TestUtils.createServer(config, mock);

    // create topics        
    AdminUtils.createTopic(zkUtils, TOPIC_OOS, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

}
 
Example #9
Source File: KafkaTopicConfig.java    From nakadi with MIT License 6 votes vote down vote up
public KafkaTopicConfig(final String topicName,
                        final int partitionCount,
                        final int replicaFactor,
                        final String cleanupPolicy,
                        final long segmentMs,
                        final Optional<Long> retentionMs,
                        final Optional<Long> segmentBytes,
                        final Optional<Long> minCompactionLagMs,
                        final RackAwareMode rackAwareMode) {
    this.topicName = topicName;
    this.partitionCount = partitionCount;
    this.replicaFactor = replicaFactor;
    this.cleanupPolicy = cleanupPolicy;
    this.segmentMs = segmentMs;
    this.retentionMs = retentionMs;
    this.segmentBytes = segmentBytes;
    this.minCompactionLagMs = minCompactionLagMs;
    this.rackAwareMode = rackAwareMode;
}
 
Example #10
Source File: KafkaStoreUtils.java    From data-highway with Apache License 2.0 6 votes vote down vote up
public static void checkAndCreateTopic(String zkConnect, String topic, int replicas) {
  ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkConnect), IS_SECURE_KAFKA_CLUSTER);

  if (AdminUtils.topicExists(zkUtils, topic)) {
    verifyTopic(zkUtils, topic);
    return;
  }

  int partitions = 1;
  Properties topicConfig = new Properties();
  topicConfig.put(LogConfig.CleanupPolicyProp(), "compact");

  AdminUtils.createTopic(zkUtils, topic, partitions, replicas, topicConfig, RackAwareMode.Enforced$.MODULE$);

  zkClient.close();
  zkUtils.close();
}
 
Example #11
Source File: KafkaEmbedded.java    From ksql-fork-with-deep-learning-function with Apache License 2.0 6 votes vote down vote up
/**
 * Create a Kafka topic with the given parameters.
 *
 * @param topic       The name of the topic.
 * @param partitions  The number of partitions for this topic.
 * @param replication The replication factor for (partitions of) this topic.
 * @param topicConfig Additional topic-level configuration settings.
 */
public void createTopic(String topic,
                        int partitions,
                        int replication,
                        Properties topicConfig) {
  log.debug("Creating topic { name: {}, partitions: {}, replication: {}, config: {} }",
      topic, partitions, replication, topicConfig);
  // Note: You must initialize the ZkClient with ZKStringSerializer.  If you don't, then
  // registerTopic() will only seem to work (it will return without error).  The topic will exist in
  // only ZooKeeper and will be returned when listing topics, but Kafka itself does not create the
  // topic.
  ZkClient zkClient = new ZkClient(
      zookeeperConnect(),
      DEFAULT_ZK_SESSION_TIMEOUT_MS,
      DEFAULT_ZK_CONNECTION_TIMEOUT_MS,
      ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zookeeperConnect()), false);
  AdminUtils.createTopic(zkUtils, topic, partitions, replication, topicConfig, RackAwareMode.Enforced$.MODULE$);
  zkClient.close();
}
 
Example #12
Source File: KafkaServiceImplTest.java    From metron with Apache License 2.0 5 votes vote down vote up
@Test
public void whenAdminUtilsThrowsAdminOperationExceptionCreateTopicShouldProperlyWrapExceptionInRestException() {
  final Map<String, List<PartitionInfo>> topics = new HashMap<>();
  topics.put("1", new ArrayList<>());

  when(kafkaConsumer.listTopics()).thenReturn(topics);

  doThrow(AdminOperationException.class).when(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$));

  assertThrows(RestException.class, () -> kafkaService.createTopic(VALID_KAFKA_TOPIC));
}
 
Example #13
Source File: KafkaServiceImpl.java    From metron with Apache License 2.0 5 votes vote down vote up
@Override
public KafkaTopic createTopic(final KafkaTopic topic) throws RestException {
  if (!listTopics().contains(topic.getName())) {
    try {
      adminUtils.createTopic(zkUtils, topic.getName(), topic.getNumPartitions(), topic.getReplicationFactor(), topic.getProperties(), RackAwareMode.Disabled$.MODULE$);
      if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)){
        addACLToCurrentUser(topic.getName());
      }
    } catch (AdminOperationException e) {
      throw new RestException(e);
    }
  }
  return topic;
}
 
Example #14
Source File: KafkaTestInstanceRule.java    From rya with Apache License 2.0 5 votes vote down vote up
/**
 * Utility method to provide additional unique topics if they are required.
 * @param topicName - The Kafka topic to create.
 */
public void createTopic(final String topicName) {
    // Setup Kafka.
    ZkUtils zkUtils = null;
    try {
        logger.info("Creating Kafka Topic: '{}'", topicName);
        zkUtils = ZkUtils.apply(new ZkClient(kafkaInstance.getZookeeperConnect(), 30000, 30000, ZKStringSerializer$.MODULE$), false);
        AdminUtils.createTopic(zkUtils, topicName, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    }
    finally {
        if(zkUtils != null) {
            zkUtils.close();
        }
    }
}
 
Example #15
Source File: KafkaConsumer.java    From cubeai with Apache License 2.0 5 votes vote down vote up
private void createKafkaTopics() {
    log.info("--------------------------------------------------------------------------");
    log.info("-----------------------Begin to create Kafka topics-----------------------");

    ZkUtils zkUtils = ZkUtils.apply(zkNodes + ":2181", 30000, 30000, JaasUtils.isZkSecurityEnabled());

    if (!AdminUtils.topicExists(zkUtils, "async-task-topic")) {
        AdminUtils.createTopic(zkUtils, "async-task-topic", 1, 1,  new Properties(), new RackAwareMode.Enforced$());
    }

    zkUtils.close();

    log.info("-----------------------Kafka topics created-------------------------------");
    log.info("--------------------------------------------------------------------------");
}
 
Example #16
Source File: ZookeeperUtils.java    From kafka-workers with Apache License 2.0 5 votes vote down vote up
public static void createTopics(String zookeeperUrl, int partitions, int replicas, String... topicNames) throws InterruptedException {

        ZkClient zkClient = ZkUtils.createZkClient(zookeeperUrl, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS);
        ZkConnection zkConnection = new ZkConnection(zookeeperUrl);
        ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);

        for (String topicName : topicNames) {
            AdminUtils.createTopic(zkUtils, topicName, partitions, replicas, new Properties(), RackAwareMode.Enforced$.MODULE$);
        }

        zkUtils.close();
        zkConnection.close();
        zkClient.close();
    }
 
Example #17
Source File: KafkaTestHelper.java    From nakadi with MIT License 5 votes vote down vote up
public void createTopic(final String topic, final String zkUrl) {
    try (KafkaZkClient zkClient = createZkClient(zkUrl)) {
        final AdminZkClient adminZkClient = new AdminZkClient(zkClient);
        adminZkClient.createTopic(topic, 1, 1,
                new Properties(), RackAwareMode.Safe$.MODULE$);
    }
}
 
Example #18
Source File: KafkaConsumer.java    From cubeai with Apache License 2.0 5 votes vote down vote up
private void createKafkaTopics() {
    log.info("--------------------------------------------------------------------------");
    log.info("-----------------------Begin to create Kafka topics-----------------------");

    ZkUtils zkUtils = ZkUtils.apply(zkNodes + ":2181", 30000, 30000, JaasUtils.isZkSecurityEnabled());

    if (!AdminUtils.topicExists(zkUtils, "async-task-topic")) {
        AdminUtils.createTopic(zkUtils, "async-task-topic", 1, 1,  new Properties(), new RackAwareMode.Enforced$());
    }

    zkUtils.close();

    log.info("-----------------------Kafka topics created-------------------------------");
    log.info("--------------------------------------------------------------------------");
}
 
Example #19
Source File: KafkaTopicConfigFactory.java    From nakadi with MIT License 5 votes vote down vote up
public KafkaTopicConfig createKafkaTopicConfig(final NakadiTopicConfig topicConfig) throws TopicConfigException {

        // set common values
        final KafkaTopicConfigBuilder configBuilder = KafkaTopicConfigBuilder.builder()
                .withTopicName(uuidGenerator.randomUUID().toString())
                .withPartitionCount(topicConfig.getPartitionCount())
                .withReplicaFactor(defaultTopicReplicaFactor)
                .withRackAwareMode(RackAwareMode.Safe$.MODULE$);

        if (topicConfig.getCleanupPolicy() == CleanupPolicy.COMPACT) {
            // set values specific for cleanup policy 'compact'
            configBuilder
                    .withCleanupPolicy("compact")
                    .withSegmentMs(compactedTopicRotationMs)
                    .withSegmentBytes(compactedTopicSegmentBytes)
                    .withMinCompactionLagMs(compactedTopicCompactionLagMs);

        } else if (topicConfig.getCleanupPolicy() == CleanupPolicy.DELETE) {
            // set values specific for cleanup policy 'delete'
            configBuilder
                    .withCleanupPolicy("delete")
                    .withRetentionMs(topicConfig.getRetentionTimeMs()
                            .orElseThrow(() -> new TopicConfigException("retention time should be specified " +
                                    "for topic with cleanup policy 'delete'")))
                    .withSegmentMs(defaultTopicRotationMs);
        }
        return configBuilder.build();
    }
 
Example #20
Source File: KafkaTestUtils.java    From ranger with Apache License 2.0 5 votes vote down vote up
static void createSomeTopics(String zkConnectString) {
	ZooKeeperClient zookeeperClient = new ZooKeeperClient(zkConnectString, 30000, 30000,
			1, Time.SYSTEM, "kafka.server", "SessionExpireListener", Option.empty());
	try (KafkaZkClient kafkaZkClient = new KafkaZkClient(zookeeperClient, false, Time.SYSTEM)) {
		AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
		adminZkClient.createTopic("test", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
		adminZkClient.createTopic("dev", 1, 1, new Properties(), RackAwareMode.Enforced$.MODULE$);
	}
}
 
Example #21
Source File: AtlasTopicCreator.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
    int numPartitions = atlasProperties.getInt("atlas.notification.hook.numthreads", 1);
    int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1);
    AdminUtils.createTopic(zkUtils, topicName,  numPartitions, numReplicas,
            new Properties(), RackAwareMode.Enforced$.MODULE$);
    LOG.warn("Created topic {} with partitions {} and replicas {}", topicName, numPartitions, numReplicas);
}
 
Example #22
Source File: TollboothApp.java    From data-highway with Apache License 2.0 5 votes vote down vote up
private void checkAndCreateTopic(String zkConnect, String topic, int partitions, int replicas) {
  ZkClient zkClient = new ZkClient(zkConnect, SESSION_TIMEOUT_MS, CONNECTION_TIMEOUT_MS, ZKStringSerializer$.MODULE$);
  ZkUtils zkUtils = new ZkUtils(zkClient, new ZkConnection(zkConnect), IS_SECURE_KAFKA_CLUSTER);

  if (!AdminUtils.topicExists(zkUtils, topic)) {
    AdminUtils.createTopic(zkUtils, topic, partitions, replicas, new Properties(), RackAwareMode.Enforced$.MODULE$);
  }

  zkUtils.close();
  zkClient.close();
}
 
Example #23
Source File: TestDatastreamServer.java    From brooklin with BSD 2-Clause "Simplified" License 5 votes vote down vote up
@Test
public void testUserManagedDestination() throws Exception {
  int numberOfPartitions = 2;
  String destinationTopic = "userManaged_" + UUID.randomUUID().toString();
  _datastreamCluster =
      initializeTestDatastreamServerWithFileConnector(1, BROADCAST_STRATEGY_FACTORY, numberOfPartitions);
  int totalEvents = 10;
  _datastreamCluster.startup();

  Path tempFile1 = Files.createTempFile("testFile1", "");
  String fileName1 = tempFile1.toAbsolutePath().toString();

  ZkClient zkClient = new ZkClient(_datastreamCluster.getZkConnection());
  ZkConnection zkConnection = new ZkConnection(_datastreamCluster.getZkConnection());
  ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
  AdminUtils.createTopic(zkUtils, destinationTopic, numberOfPartitions, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);

  Datastream fileDatastream1 = createFileDatastream(fileName1, destinationTopic, 2);
  Assert.assertEquals((int) fileDatastream1.getDestination().getPartitions(), 2);

  Collection<String> eventsWritten1 = TestUtils.generateStrings(totalEvents);
  FileUtils.writeLines(new File(fileName1), eventsWritten1);

  Collection<String> eventsReceived1 = readFileDatastreamEvents(fileDatastream1, 0, 4);
  Collection<String> eventsReceived2 = readFileDatastreamEvents(fileDatastream1, 1, 6);
  eventsReceived1.addAll(eventsReceived2);

  LOG.info("Events Received " + eventsReceived1);
  LOG.info("Events Written to file " + eventsWritten1);

  Assert.assertTrue(eventsReceived1.containsAll(eventsWritten1));
}
 
Example #24
Source File: KafkaProducerServiceIntegrationTest.java    From vertx-kafka-service with Apache License 2.0 5 votes vote down vote up
private void createTopic(String topic) {
    final ZkClient zkClient = new ZkClient(zookeeper.getConnectString(), 30000, 30000, ZKStringSerializer$.MODULE$);
    final ZkConnection connection = new ZkConnection(zookeeper.getConnectString());
    final ZkUtils zkUtils = new ZkUtils(zkClient, connection, false);
    AdminUtils.createTopic(zkUtils, topic, 1, 1, new Properties(), RackAwareMode.Disabled$.MODULE$);
    TestUtils.waitUntilMetadataIsPropagated(JavaConversions.asScalaBuffer(Lists.newArrayList(kafkaServer)), topic, 0, 10000);
    zkClient.close();
}
 
Example #25
Source File: KafkaClient.java    From kafka-service-broker with Apache License 2.0 5 votes vote down vote up
void createTopic(String topicName) {
    ZkUtils zu = null;
    try {
        zu = util.getUtils();
        AdminUtils.createTopic(zu, topicName, 2, 3, new Properties(), RackAwareMode.Disabled$.MODULE$);
    } finally {
        if (zu != null) {
            zu.close();
        }
    }
}
 
Example #26
Source File: CompositeTransactionManagerKafkaImpl.java    From microservices-transactions-tcc with Apache License 2.0 5 votes vote down vote up
@Override
public void open(String txId) {
	// Add topic configuration here
       Properties topicConfig = new Properties();

       AdminUtils.createTopic(zkUtils, txId, zooPartitions, zooReplication, topicConfig, RackAwareMode.Enforced$.MODULE$);
}
 
Example #27
Source File: KafkaTestUtil.java    From siddhi-io-kafka with Apache License 2.0 5 votes vote down vote up
public static void createTopic(String connectionString, String topics[], int numOfPartitions) {
    ZkClient zkClient = new ZkClient(connectionString, 30000, 30000, ZKStringSerializer$.MODULE$);
    ZkConnection zkConnection = new ZkConnection(connectionString);
    ZkUtils zkUtils = new ZkUtils(zkClient, zkConnection, false);
    for (String topic : topics) {
        try {
            AdminUtils.createTopic(zkUtils, topic, numOfPartitions, 1, new Properties(),
                    RackAwareMode.Enforced$.MODULE$);
        } catch (TopicExistsException e) {
            log.warn("topic exists for: " + topic);
        }
    }
    zkClient.close();
}
 
Example #28
Source File: AtlasTopicCreator.java    From atlas with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
protected void createTopic(Configuration atlasProperties, String topicName, ZkUtils zkUtils) {
    int numPartitions = atlasProperties.getInt("atlas.notification.hook.numthreads", 1);
    int numReplicas = atlasProperties.getInt("atlas.notification.replicas", 1);
    AdminUtils.createTopic(zkUtils, topicName,  numPartitions, numReplicas,
            new Properties(), RackAwareMode.Enforced$.MODULE$);
    LOG.warn("Created topic {} with partitions {} and replicas {}", topicName, numPartitions, numReplicas);
}
 
Example #29
Source File: LoadMonitorTaskRunnerTest.java    From cruise-control with BSD 2-Clause "Simplified" License 5 votes vote down vote up
/**
 * Setup the test.
 */
@Before
public void setUp() {
  super.setUp();
  KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(),
                                                                            "LoadMonitorTaskRunnerGroup",
                                                                            "LoadMonitorTaskRunnerSetup",
                                                                            false);
  AdminZkClient adminZkClient = new AdminZkClient(kafkaZkClient);
  for (int i = 0; i < NUM_TOPICS; i++) {
    adminZkClient.createTopic("topic-" + i, NUM_PARTITIONS, 1, new Properties(), RackAwareMode.Safe$.MODULE$);
  }
  KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
}
 
Example #30
Source File: EmbeddedKafka.java    From tajo with Apache License 2.0 5 votes vote down vote up
public void createTopic(int partitions, int replication, String topic) {
  checkState(started.get(), "not started!");

  ZkClient zkClient = new ZkClient(getZookeeperConnectString(), 30000, 30000, ZKStringSerializer$.MODULE$);
  try {
    AdminUtils.createTopic(ZkUtils.apply(zkClient, false), topic, partitions, replication, new Properties(),
        RackAwareMode.Enforced$.MODULE$);
  } finally {
    zkClient.close();
  }
}