Java Code Examples for org.apache.pulsar.broker.ServiceConfiguration#setZookeeperServers()

The following examples show how to use org.apache.pulsar.broker.ServiceConfiguration#setZookeeperServers() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AdvertisedAddressTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@BeforeMethod
public void setup() throws Exception {
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    ServiceConfiguration config = new ServiceConfiguration();
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setWebServicePort(Optional.ofNullable(0));
    config.setClusterName("usc");
    config.setAdvertisedAddress("localhost");
    config.setBrokerServicePort(Optional.ofNullable(0));
    config.setAdvertisedAddress(advertisedAddress);
    config.setManagedLedgerMaxEntriesPerLedger(5);
    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsar = new PulsarService(config);
    pulsar.start();
}
 
Example 2
Source File: ModularLoadManagerImplTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
/**
 * It verifies that pulsar-service fails if load-manager tries to create ephemeral znode for broker which is already
 * created by other zk-session-id.
 *
 * @throws Exception
 */
@Test
public void testOwnBrokerZnodeByMultipleBroker() throws Exception {

    ServiceConfiguration config = new ServiceConfiguration();
    config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config.setClusterName("use");
    config.setWebServicePort(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    PulsarService pulsar = new PulsarService(config);
    // create znode using different zk-session
    final String brokerZnode = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getAdvertisedAddress() + ":"
            + config.getWebServicePort();
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), brokerZnode, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.EPHEMERAL);
    try {
        pulsar.start();
    } catch (PulsarServerException e) {
        //Ok.
    }

    pulsar.close();
}
 
Example 3
Source File: TopicOwnerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@BeforeMethod
void setup() throws Exception {
    log.info("---- Initializing TopicOwnerTest -----");
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {
        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.of(0));
        config.setClusterName("my-cluster");
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setDefaultNumberOfNamespaceBundles(1);
        config.setLoadBalancerEnabled(false);
        configurations[i] = config;

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();

        pulsarAdmins[i] = PulsarAdmin.builder()
                .serviceHttpUrl(pulsarServices[i].getWebServiceAddress())
                .build();
    }
    Thread.sleep(1000);
}
 
Example 4
Source File: MaxMessageSizeTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@BeforeMethod
void setup() {
    try {
        bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
        ServerConfiguration conf = new ServerConfiguration();
        conf.setNettyMaxFrameSizeBytes(10 * 1024 * 1024);
        bkEnsemble.startStandalone(conf, false);

        configuration = new ServiceConfiguration();
        configuration.setZookeeperServers("127.0.0.1:" + bkEnsemble.getZookeeperPort());
        configuration.setAdvertisedAddress("localhost");
        configuration.setWebServicePort(Optional.of(0));
        configuration.setClusterName("max_message_test");
        configuration.setBrokerServicePort(Optional.of(0));
        configuration.setAuthorizationEnabled(false);
        configuration.setAuthenticationEnabled(false);
        configuration.setManagedLedgerMaxEntriesPerLedger(5);
        configuration.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
        configuration.setMaxMessageSize(10 * 1024 * 1024);

        pulsar = new PulsarService(configuration);
        pulsar.start();

        String url = "http://127.0.0.1:" + pulsar.getListenPortHTTP().get();
        admin = PulsarAdmin.builder().serviceHttpUrl(url).build();
        admin.clusters().createCluster("max_message_test", new ClusterData(url));
        admin.tenants()
             .createTenant("test", new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("max_message_test")));
        admin.namespaces().createNamespace("test/message", Sets.newHashSet("max_message_test"));
    } catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 5
Source File: PulsarClientTest.java    From java-specialagent with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  if (!isJdkSupported)
    return;

  bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, TestUtil::nextFreePort);
  bkEnsemble.start();

  final int brokerWebServicePort = TestUtil.nextFreePort();
  final int brokerServicePort = TestUtil.nextFreePort();

  final ServiceConfiguration config = new ServiceConfiguration();
  config.setClusterName(CLUSTER_NAME);
  final Set<String> superUsers = Sets.newHashSet("superUser");
  config.setSuperUserRoles(superUsers);
  config.setWebServicePort(brokerWebServicePort);
  config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
  config.setBrokerServicePort(brokerServicePort);
  config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
  config.setTlsAllowInsecureConnection(true);
  config.setAdvertisedAddress("localhost");

  pulsarService = new PulsarService(config);
  pulsarService.start();

  try (final PulsarAdmin admin = pulsarService.getAdminClient()) {
    final ClusterData clusterData = new ClusterData(pulsarService.getBrokerServiceUrl());
    admin.clusters().createCluster(CLUSTER_NAME, clusterData);

    final TenantInfo propAdmin = new TenantInfo();
    propAdmin.getAdminRoles().add("superUser");
    propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList(CLUSTER_NAME)));

    admin.tenants().createTenant("public", propAdmin);
    admin.namespaces().createNamespace("public/default", Sets.newHashSet(CLUSTER_NAME));
  }
}
 
Example 6
Source File: BrokerAdminClientTlsAuthTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * Test case => Use Multiple Brokers
 *           => Create a namespace with bundles distributed among these brokers.
 *           => Use Tls as authPlugin for everything.
 *           => Run list topics command
 * @throws Exception
 */
@Test
public void testPersistentList() throws Exception {
    log.info("-- Starting {} test --", methodName);

    /***** Start Broker 2 ******/
    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setBrokerServicePort(Optional.of(0));
    conf.setBrokerServicePortTls(Optional.of(0));
    conf.setWebServicePort(Optional.of(0));
    conf.setWebServicePortTls(Optional.of(0));
    conf.setAdvertisedAddress("localhost");
    conf.setClusterName(this.conf.getClusterName());
    conf.setZookeeperServers("localhost:2181");
    buildConf(conf);

    @Cleanup
    PulsarService pulsar2 = startBroker(conf);

    /***** Broker 2 Started *****/
    try (PulsarAdmin admin = buildAdminClient("superproxy")) {
        admin.clusters().createCluster("test", new ClusterData(brokerUrl.toString()));
        admin.tenants().createTenant("tenant",
                                     new TenantInfo(ImmutableSet.of("admin"),
                                                    ImmutableSet.of("test")));
    }
    try (PulsarAdmin admin = buildAdminClient("admin")) {
        Policies policies = new Policies();
        policies.bundles = new BundlesData(4);
        policies.auth_policies.namespace_auth.put("admin", ImmutableSet.of(AuthAction.produce, AuthAction.consume));
        policies.replication_clusters = ImmutableSet.of("test");
        admin.namespaces().createNamespace("tenant/ns", policies);
        try {
            admin.topics().getList("tenant/ns");
        } catch (PulsarAdminException ex) {
            ex.printStackTrace();
            fail("Should not have thrown an exception");
        }
    }

}
 
Example 7
Source File: BrokerBookieIsolationTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@Test
public void testDeleteIsolationGroup() throws Exception {

    final String tenant1 = "tenant1";
    final String cluster = "use";
    final String ns2 = String.format("%s/%s/%s", tenant1, cluster, "ns2");
    final String ns3 = String.format("%s/%s/%s", tenant1, cluster, "ns3");

    final String brokerBookkeeperClientIsolationGroups = "default-group";
    final String tenantNamespaceIsolationGroupsPrimary = "tenant1-isolation-primary";
    final String tenantNamespaceIsolationGroupsSecondary = "tenant1-isolation=secondary";

    BookieServer[] bookies = bkEnsemble.getBookies();
    ZooKeeper zkClient = bkEnsemble.getZkClient();

    Set<BookieSocketAddress> defaultBookies = Sets.newHashSet(bookies[0].getLocalAddress(),
            bookies[1].getLocalAddress());
    Set<BookieSocketAddress> isolatedBookies = Sets.newHashSet(bookies[2].getLocalAddress(),
            bookies[3].getLocalAddress());

    setDefaultIsolationGroup(brokerBookkeeperClientIsolationGroups, zkClient, defaultBookies);
    // primary group empty
    setDefaultIsolationGroup(tenantNamespaceIsolationGroupsPrimary, zkClient, Sets.newHashSet());
    setDefaultIsolationGroup(tenantNamespaceIsolationGroupsSecondary, zkClient, isolatedBookies);

    ServiceConfiguration config = new ServiceConfiguration();
    config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config.setClusterName(cluster);
    config.setWebServicePort(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    config.setAdvertisedAddress("localhost");
    config.setBookkeeperClientIsolationGroups(brokerBookkeeperClientIsolationGroups);

    config.setManagedLedgerDefaultEnsembleSize(2);
    config.setManagedLedgerDefaultWriteQuorum(2);
    config.setManagedLedgerDefaultAckQuorum(2);
    config.setAllowAutoTopicCreationType("non-partitioned");

    config.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
    pulsarService = new PulsarService(config);
    pulsarService.start();

    PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(pulsarService.getWebServiceAddress()).build();

    ClusterData clusterData = new ClusterData(pulsarService.getWebServiceAddress());
    admin.clusters().createCluster(cluster, clusterData);
    TenantInfo tenantInfo = new TenantInfo(null, Sets.newHashSet(cluster));
    admin.tenants().createTenant(tenant1, tenantInfo);
    admin.namespaces().createNamespace(ns2);
    admin.namespaces().createNamespace(ns3);

    // (1) set affinity-group
    admin.namespaces().setBookieAffinityGroup(ns2, new BookieAffinityGroupData(
            tenantNamespaceIsolationGroupsPrimary, tenantNamespaceIsolationGroupsSecondary));
    admin.namespaces().setBookieAffinityGroup(ns3, new BookieAffinityGroupData(
            tenantNamespaceIsolationGroupsPrimary, tenantNamespaceIsolationGroupsSecondary));

    // (2) get affinity-group
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns2), new BookieAffinityGroupData(
            tenantNamespaceIsolationGroupsPrimary, tenantNamespaceIsolationGroupsSecondary));
    assertEquals(admin.namespaces().getBookieAffinityGroup(ns3), new BookieAffinityGroupData(
            tenantNamespaceIsolationGroupsPrimary, tenantNamespaceIsolationGroupsSecondary));

    // (3) delete affinity-group
    admin.namespaces().deleteBookieAffinityGroup(ns2);

    try {
        admin.namespaces().getBookieAffinityGroup(ns2);
        fail("should have fail due to affinity-group not present");
    } catch (NotFoundException e) {
        // Ok
    }

    assertEquals(admin.namespaces().getBookieAffinityGroup(ns3), new BookieAffinityGroupData(
            tenantNamespaceIsolationGroupsPrimary, tenantNamespaceIsolationGroupsSecondary));

}
 
Example 8
Source File: SimpleLoadManagerImplTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@BeforeMethod
void setup() throws Exception {

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = spy(new ServiceConfiguration());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config1.setBrokerServicePort(Optional.of(0));
    config1.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config1.setBrokerServicePortTls(Optional.of(0));
    config1.setWebServicePortTls(Optional.of(0));
    config1.setAdvertisedAddress("localhost");
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    url1 = new URL(pulsar1.getWebServiceAddress());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();
    brokerStatsClient1 = admin1.brokerStats();
    primaryHost = pulsar1.getWebServiceAddress();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setBrokerServicePort(Optional.of(0));
    config2.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
    config2.setBrokerServicePortTls(Optional.of(0));
    config2.setWebServicePortTls(Optional.of(0));
    config2.setAdvertisedAddress("localhost");
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    url2 = new URL(pulsar2.getWebServiceAddress());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();
    brokerStatsClient2 = admin2.brokerStats();
    secondaryHost = pulsar2.getWebServiceAddress();
    Thread.sleep(100);
}
 
Example 9
Source File: LoadBalancerTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@BeforeMethod
void setup() throws Exception {
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();
    ZkUtils.createFullPathOptimistic(bkEnsemble.getZkClient(),
            SimpleLoadManagerImpl.LOADBALANCER_DYNAMIC_SETTING_STRATEGY_ZPATH,
            "{\"loadBalancerStrategy\":\"leastLoadedServer\"}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);

    final String localhost = "localhost";
    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {


        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.ofNullable(brokerNativeBrokerPorts[i]));
        config.setClusterName("use");
        config.setAdvertisedAddress(localhost);
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setBrokerServicePortTls(Optional.of(0));
        config.setWebServicePortTls(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setBrokerServicePort(Optional.of(0));
        config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
        config.setAdvertisedAddress(localhost+i);
        config.setLoadBalancerEnabled(false);

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();
        brokerWebServicePorts[i] = pulsarServices[i].getListenPortHTTP().get();
        brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get();

        brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
        lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + pulsarServices[i].getListenPortHTTP().get();
        pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build();
    }

    createNamespacePolicies(pulsarServices[0]);

    Thread.sleep(100);
}
 
Example 10
Source File: ModularLoadManagerImplTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@BeforeMethod
void setup() throws Exception {
    executor = new ThreadPoolExecutor(1, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>());

    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());

    config1.setAdvertisedAddress("localhost");
    config1.setBrokerServicePort(Optional.of(0));
    config1.setBrokerServicePortTls(Optional.of(0));
    config1.setWebServicePortTls(Optional.of(0));
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
    url1 = new URL(pulsar1.getWebServiceAddress());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setAdvertisedAddress("localhost");
    config2.setBrokerServicePort(Optional.of(0));
    config2.setBrokerServicePortTls(Optional.of(0));
    config2.setWebServicePortTls(Optional.of(0));
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());
    url2 = new URL(pulsar2.getWebServiceAddress());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();

    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}
 
Example 11
Source File: AntiAffinityNamespaceGroupTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@BeforeMethod
void setup() throws Exception {
    executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS,
            new LinkedBlockingQueue<Runnable>());
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    // Start broker 1
    ServiceConfiguration config1 = new ServiceConfiguration();
    config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config1.setClusterName("use");
    config1.setWebServicePort(Optional.of(0));
    config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config1.setBrokerServicePort(Optional.of(0));
    config1.setFailureDomainsEnabled(true);
    config1.setLoadBalancerEnabled(true);
    config1.setAdvertisedAddress("localhost");
    createCluster(bkEnsemble.getZkClient(), config1);
    pulsar1 = new PulsarService(config1);
    pulsar1.start();

    primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get());
    url1 = new URL("http://127.0.0.1" + ":" + pulsar1.getListenPortHTTP().get());
    admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build();

    // Start broker 2
    ServiceConfiguration config2 = new ServiceConfiguration();
    config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config2.setClusterName("use");
    config2.setWebServicePort(Optional.of(0));
    config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config2.setBrokerServicePort(Optional.of(0));
    config2.setFailureDomainsEnabled(true);
    config2.setAdvertisedAddress("localhost");
    pulsar2 = new PulsarService(config2);
    pulsar2.start();

    secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get());

    url2 = new URL("http://127.0.0.1" + ":" + config2.getWebServicePort().get());
    admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build();

    primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager");
    secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager");
    nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32());
    Thread.sleep(100);
}
 
Example 12
Source File: TransactionMetaStoreTestBase.java    From pulsar with Apache License 2.0 4 votes vote down vote up
@BeforeClass
void setup() throws Exception {
    log.info("---- Initializing SLAMonitoringTest -----");
    // Start local bookkeeper ensemble
    bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0);
    bkEnsemble.start();

    String[] args = new String[]{
        "--cluster", "my-cluster",
        "--configuration-store", "localhost:" + bkEnsemble.getZookeeperPort(),
        "--initial-num-transaction-coordinators", "16"};

    PulsarTransactionCoordinatorMetadataSetup.main(args);

    // start brokers
    for (int i = 0; i < BROKER_COUNT; i++) {
        ServiceConfiguration config = new ServiceConfiguration();
        config.setBrokerServicePort(Optional.of(0));
        config.setClusterName("my-cluster");
        config.setAdvertisedAddress("localhost");
        config.setWebServicePort(Optional.of(0));
        config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
        config.setDefaultNumberOfNamespaceBundles(1);
        config.setLoadBalancerEnabled(false);
        configurations[i] = config;

        pulsarServices[i] = new PulsarService(config);
        pulsarServices[i].start();

        pulsarAdmins[i] = PulsarAdmin.builder()
                .serviceHttpUrl(pulsarServices[i].getWebServiceAddress())
                .build();
    }

    Thread.sleep(100);

    PulsarClient client = PulsarClient.builder().
        serviceUrl(pulsarServices[0].getBrokerServiceUrl())
        .build();
    transactionCoordinatorClient = new TransactionCoordinatorClientImpl(client);
    transactionCoordinatorClient.start();

    Thread.sleep(3000);
}
 
Example 13
Source File: WebServiceTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
private void setupEnv(boolean enableFilter, String minApiVersion, boolean allowUnversionedClients,
        boolean enableTls, boolean enableAuth, boolean allowInsecure) throws Exception {
    Set<String> providers = new HashSet<>();
    providers.add("org.apache.pulsar.broker.authentication.AuthenticationProviderTls");

    Set<String> roles = new HashSet<>();
    roles.add("client");

    ServiceConfiguration config = new ServiceConfiguration();
    config.setAdvertisedAddress("localhost");
    config.setBrokerServicePort(Optional.of(0));
    config.setWebServicePort(Optional.of(0));
    if (enableTls) {
        config.setWebServicePortTls(Optional.of(0));
    }
    config.setClientLibraryVersionCheckEnabled(enableFilter);
    config.setAuthenticationEnabled(enableAuth);
    config.setAuthenticationProviders(providers);
    config.setAuthorizationEnabled(false);
    config.setSuperUserRoles(roles);
    config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    config.setTlsAllowInsecureConnection(allowInsecure);
    config.setTlsTrustCertsFilePath(allowInsecure ? "" : TLS_CLIENT_CERT_FILE_PATH);
    config.setClusterName("local");
    config.setAdvertisedAddress("localhost"); // TLS certificate expects localhost
    config.setZookeeperServers("localhost:2181");
    config.setHttpMaxRequestSize(10 * 1024);
    pulsar = spy(new PulsarService(config));
    doReturn(zkFactory).when(pulsar).getZooKeeperClientFactory();
    doReturn(new MockedBookKeeperClientFactory()).when(pulsar).newBookKeeperClientFactory();
    pulsar.start();

    try {
        pulsar.getZkClient().delete("/minApiVersion", -1);
    } catch (Exception ex) {
    }
    pulsar.getZkClient().create("/minApiVersion", minApiVersion.getBytes(), null, CreateMode.PERSISTENT);

    String BROKER_URL_BASE = "http://localhost:" + pulsar.getListenPortHTTP().get();
    String BROKER_URL_BASE_TLS = "https://localhost:" + pulsar.getListenPortHTTPS().orElse(-1);
    String serviceUrl = BROKER_URL_BASE;

    PulsarAdminBuilder adminBuilder = PulsarAdmin.builder();
    if (enableTls && enableAuth) {
        serviceUrl = BROKER_URL_BASE_TLS;

        Map<String, String> authParams = new HashMap<>();
        authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH);
        authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH);

        adminBuilder.authentication(AuthenticationTls.class.getName(), authParams).allowTlsInsecureConnection(true);
    }

    BROKER_LOOKUP_URL = BROKER_URL_BASE
            + "/lookup/v2/destination/persistent/my-property/local/my-namespace/my-topic";
    BROKER_LOOKUP_URL_TLS = BROKER_URL_BASE_TLS
            + "/lookup/v2/destination/persistent/my-property/local/my-namespace/my-topic";

    PulsarAdmin pulsarAdmin = adminBuilder.serviceHttpUrl(serviceUrl).build();

    try {
        pulsarAdmin.clusters().createCluster(config.getClusterName(),
                new ClusterData(pulsar.getSafeWebServiceAddress()));
    } catch (ConflictException ce) {
        // This is OK.
    } finally {
        pulsarAdmin.close();
    }
}
 
Example 14
Source File: BrokerServiceLookupTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 *
 * <pre>
 * When broker-1's load-manager splits the bundle and update local-policies, broker-2 should get watch of
 * local-policies and update bundleCache so, new lookup can be redirected properly.
 *
 * (1) Start broker-1 and broker-2
 * (2) Make sure broker-2 always assign bundle to broker1
 * (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
 * (4) Broker-1 will own topic-1
 * (5) Split the bundle for topic-1
 * (6) Broker-2 should get the watch and update bundle cache
 * (7) Make lookup request again to Broker-2 which should succeed.
 *
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testSplitUnloadLookupTest() throws Exception {

    log.info("-- Starting {} test --", methodName);

    final String namespace = "my-property/my-ns";
    // (1) Start broker-1
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(Optional.of(0));
    conf2.setWebServicePort(Optional.of(0));
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");

    @Cleanup
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();

    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();

    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);

    // (2) Make sure broker-2 always assign bundle to broker1
    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    // mock: return Broker1 as a Least-loaded broker when leader receives request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getSafeWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));

    @Cleanup
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(pulsar2.getBrokerServiceUrl()).build();

    // (3) Broker-2 receives topic-1 request, creates local-policies and sets the watch
    final String topic1 = "persistent://" + namespace + "/topic1";
    Consumer<byte[]> consumer1 = pulsarClient2.newConsumer().topic(topic1).subscriptionName("my-subscriber-name")
            .subscribe();

    Set<String> serviceUnits1 = pulsar.getNamespaceService().getOwnedServiceUnits().stream()
            .map(nb -> nb.toString()).collect(Collectors.toSet());

    // (4) Broker-1 will own topic-1
    final String unsplitBundle = namespace + "/0x00000000_0xffffffff";
    assertTrue(serviceUnits1.contains(unsplitBundle));
    // broker-2 should have this bundle into the cache
    TopicName topicName = TopicName.get(topic1);
    NamespaceBundle bundleInBroker2 = pulsar2.getNamespaceService().getBundle(topicName);
    assertEquals(bundleInBroker2.toString(), unsplitBundle);

    // (5) Split the bundle for topic-1
    admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true, null);

    // (6) Broker-2 should get the watch and update bundle cache
    final int retry = 5;
    for (int i = 0; i < retry; i++) {
        if (pulsar2.getNamespaceService().getBundle(topicName).equals(bundleInBroker2) && i != retry - 1) {
            Thread.sleep(200);
        } else {
            break;
        }
    }

    // (7) Make lookup request again to Broker-2 which should succeed.
    final String topic2 = "persistent://" + namespace + "/topic2";
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topic2).subscriptionName("my-subscriber-name")
            .subscribe();

    NamespaceBundle bundleInBroker1AfterSplit = pulsar2.getNamespaceService().getBundle(TopicName.get(topic2));
    assertNotEquals(unsplitBundle, bundleInBroker1AfterSplit);

    consumer1.close();
    consumer2.close();
}
 
Example 15
Source File: BrokerServiceLookupTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * 1. Start broker1 and broker2 with tls enable 2. Hit HTTPS lookup url at broker2 which redirects to HTTPS broker1
 *
 * @throws Exception
 */
@Test
public void testWebserviceServiceTls() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
    final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
    final String TLS_CLIENT_CERT_FILE_PATH = "./src/test/resources/certificate/client.crt";
    final String TLS_CLIENT_KEY_FILE_PATH = "./src/test/resources/certificate/client.key";

    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(Optional.of(0));
    conf2.setBrokerServicePortTls(Optional.of(0));
    conf2.setWebServicePort(Optional.of(0));
    conf2.setWebServicePortTls(Optional.of(0));
    conf2.setAdvertisedAddress("localhost");
    conf2.setTlsAllowInsecureConnection(true);
    conf2.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf2.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");

    @Cleanup
    PulsarService pulsar2 = startBroker(conf2);

    // restart broker1 with tls enabled
    conf.setBrokerServicePortTls(Optional.of(0));
    conf.setWebServicePortTls(Optional.of(0));
    conf.setTlsAllowInsecureConnection(true);
    conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
    conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
    stopBroker();
    startBroker();
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();

    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);

    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));

    // mock: return Broker2 as a Least-loaded broker when leader receives
    // request [3]
    doReturn(true).when(loadManager1).isCentralized();
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar.getWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));


    /**** started broker-2 ****/

    URI brokerServiceUrl = new URI("pulsar://localhost:" + conf2.getBrokerServicePort().get());
    @Cleanup
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();

    final String lookupResourceUrl = "/lookup/v2/topic/persistent/my-property/my-ns/my-topic1";

    // set client cert_key file
    KeyManager[] keyManagers = null;
    Certificate[] tlsCert = SecurityUtility.loadCertificatesFromPemFile(TLS_CLIENT_CERT_FILE_PATH);
    PrivateKey tlsKey = SecurityUtility.loadPrivateKeyFromPemFile(TLS_CLIENT_KEY_FILE_PATH);
    KeyStore ks = KeyStore.getInstance(KeyStore.getDefaultType());
    ks.load(null, null);
    ks.setKeyEntry("private", tlsKey, "".toCharArray(), tlsCert);
    KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm());
    kmf.init(ks, "".toCharArray());
    keyManagers = kmf.getKeyManagers();
    TrustManager[] trustManagers = InsecureTrustManagerFactory.INSTANCE.getTrustManagers();
    SSLContext sslCtx = SSLContext.getInstance("TLS");
    sslCtx.init(keyManagers, trustManagers, new SecureRandom());
    HttpsURLConnection.setDefaultSSLSocketFactory(sslCtx.getSocketFactory());

    // hit broker2 url
    URLConnection con = new URL(pulsar2.getWebServiceAddressTls() + lookupResourceUrl).openConnection();
    log.info("orignal url: {}", con.getURL());
    con.connect();
    log.info("connected url: {} ", con.getURL());
    // assert connect-url: broker2-https
    assertEquals(new Integer(con.getURL().getPort()), conf2.getWebServicePortTls().get());
    InputStream is = con.getInputStream();
    // assert redirect-url: broker1-https only
    log.info("redirected url: {}", con.getURL());
    assertEquals(new Integer(con.getURL().getPort()), conf.getWebServicePortTls().get());
    is.close();

    loadManager1 = null;
    loadManager2 = null;
}
 
Example 16
Source File: BrokerServiceLookupTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * Create #PartitionedTopic and let it served by multiple brokers which requires a. tcp partitioned-metadata-lookup
 * b. multiple topic-lookup c. partitioned producer-consumer
 *
 * @throws Exception
 */
@Test
public void testPartitionTopicLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);

    int numPartitions = 8;
    TopicName topicName = TopicName.get("persistent://my-property/my-ns/my-partitionedtopic1");

    admin.topics().createPartitionedTopic(topicName.toString(), numPartitions);

    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(Optional.of(0));
    conf2.setWebServicePort(Optional.of(0));
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(pulsar.getConfiguration().getClusterName());
    conf2.setZookeeperServers("localhost:2181");

    @Cleanup
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();

    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);

    // mock: return Broker2 as a Least-loaded broker when leader receives request
    doReturn(true).when(loadManager1).isCentralized();
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));

    // mock: redirect request to leader
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));
    /**** broker-2 started ****/

    Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
        .topic(topicName.toString())
        .enableBatching(false)
        .messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();

    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName.toString())
            .subscriptionName("my-partitioned-subscriber").subscribe();

    for (int i = 0; i < 20; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 20; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg, "Message should not be null");
        consumer.acknowledge(msg);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        assertTrue(messageSet.add(receivedMessage), "Message " + receivedMessage + " already received");
    }

    producer.close();
    consumer.unsubscribe();
    consumer.close();
    admin.topics().deletePartitionedTopic(topicName.toString());

    loadManager2 = null;

    log.info("-- Exiting {} test --", methodName);
}
 
Example 17
Source File: BrokerServiceLookupTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * Usecase: Redirection due to different cluster 1. Broker1 runs on cluster: "use" and Broker2 runs on cluster:
 * "use2" 2. Broker1 receives "use2" cluster request => Broker1 reads "/clusters" from global-zookeeper and
 * redirects request to Broker2 which serves "use2" 3. Broker2 receives redirect request and own namespace bundle
 *
 * @throws Exception
 */
@Test(enabled = false) // See https://github.com/apache/pulsar/issues/5437
public void testMultipleBrokerDifferentClusterLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);

    /**** start broker-2 ****/
    final String newCluster = "use2";
    final String property = "my-property2";
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setAdvertisedAddress("localhost");
    conf2.setBrokerServicePort(Optional.of(0));
    conf2.setWebServicePort(Optional.of(0));
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(newCluster); // Broker2 serves newCluster
    conf2.setZookeeperServers("localhost:2181");
    String broker2ServiceUrl = "pulsar://localhost:" + conf2.getBrokerServicePort().get();

    admin.clusters().createCluster(newCluster,
            new ClusterData(pulsar.getWebServiceAddress(), null, broker2ServiceUrl, null));
    admin.tenants().createTenant(property,
            new TenantInfo(Sets.newHashSet("appid1", "appid2"), Sets.newHashSet(newCluster)));
    admin.namespaces().createNamespace(property + "/" + newCluster + "/my-ns");

    @Cleanup
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();

    URI brokerServiceUrl = new URI(broker2ServiceUrl);
    @Cleanup
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(brokerServiceUrl.toString()).build();

    // enable authorization: so, broker can validate cluster and redirect if finds different cluster
    pulsar.getConfiguration().setAuthorizationEnabled(true);
    // restart broker with authorization enabled: it initialize AuthorizationService
    stopBroker();
    startBroker();

    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);

    // mock: return Broker2 as a Least-loaded broker when leader receives request
    doReturn(true).when(loadManager2).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getSafeWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager2));
    /**** started broker-2 ****/

    // load namespace-bundle by calling Broker2
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property2/use2/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = pulsarClient2.newProducer(Schema.BYTES)
        .topic("persistent://my-property2/use2/my-ns/my-topic1")
        .create();

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();

    // disable authorization
    pulsar.getConfiguration().setAuthorizationEnabled(false);
    loadManager2 = null;
}
 
Example 18
Source File: BrokerServiceLookupTest.java    From pulsar with Apache License 2.0 4 votes vote down vote up
/**
 * Usecase Multiple Broker => Lookup Redirection test
 *
 * 1. Broker1 is a leader 2. Lookup request reaches to Broker2 which redirects to leader (Broker1) with
 * authoritative = false 3. Leader (Broker1) finds out least loaded broker as Broker2 and redirects request to
 * Broker2 with authoritative = true 4. Broker2 receives final request to own a bundle with authoritative = true and
 * client connects to Broker2
 *
 * @throws Exception
 */
@Test
public void testMultipleBrokerLookup() throws Exception {
    log.info("-- Starting {} test --", methodName);

    /**** start broker-2 ****/
    ServiceConfiguration conf2 = new ServiceConfiguration();
    conf2.setBrokerServicePort(Optional.of(0));
    conf2.setWebServicePort(Optional.of(0));
    conf2.setAdvertisedAddress("localhost");
    conf2.setClusterName(conf.getClusterName());
    conf2.setZookeeperServers("localhost:2181");

    @Cleanup
    PulsarService pulsar2 = startBroker(conf2);
    pulsar.getLoadManager().get().writeLoadReportOnZookeeper();
    pulsar2.getLoadManager().get().writeLoadReportOnZookeeper();

    LoadManager loadManager1 = spy(pulsar.getLoadManager().get());
    LoadManager loadManager2 = spy(pulsar2.getLoadManager().get());
    Field loadManagerField = NamespaceService.class.getDeclaredField("loadManager");
    loadManagerField.setAccessible(true);

    // mock: redirect request to leader [2]
    doReturn(true).when(loadManager2).isCentralized();
    loadManagerField.set(pulsar2.getNamespaceService(), new AtomicReference<>(loadManager2));

    // mock: return Broker2 as a Least-loaded broker when leader receives request [3]
    doReturn(true).when(loadManager1).isCentralized();
    SimpleResourceUnit resourceUnit = new SimpleResourceUnit(pulsar2.getSafeWebServiceAddress(), null);
    doReturn(Optional.of(resourceUnit)).when(loadManager1).getLeastLoaded(any(ServiceUnitId.class));
    doReturn(Optional.of(resourceUnit)).when(loadManager2).getLeastLoaded(any(ServiceUnitId.class));
    loadManagerField.set(pulsar.getNamespaceService(), new AtomicReference<>(loadManager1));

    /**** started broker-2 ****/

    @Cleanup
    PulsarClient pulsarClient2 = PulsarClient.builder().serviceUrl(pulsar2.getBrokerServiceUrl()).build();

    // load namespace-bundle by calling Broker2
    Consumer<byte[]> consumer = pulsarClient2.newConsumer().topic("persistent://my-property/my-ns/my-topic1")
            .subscriptionName("my-subscriber-name").subscribe();
    Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
        .topic("persistent://my-property/my-ns/my-topic1")
        .create();

    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }

    Message<byte[]> msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    producer.close();

    loadManager1 = null;
    loadManager2 = null;

}
 
Example 19
Source File: PulsarFunctionsITest.java    From java-specialagent with Apache License 2.0 4 votes vote down vote up
static void start() throws Exception {
  // Start local bookkeeper ensemble
  final LocalBookkeeperEnsemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, TestUtil::nextFreePort);
  bkEnsemble.start();

  final String brokerServiceUrl = "http://127.0.0.1:" + brokerWebServicePort;

  final ServiceConfiguration config = new ServiceConfiguration();
  config.setClusterName(CLUSTER_NAME);
  final Set<String> superUsers = Sets.newHashSet("superUser");
  config.setSuperUserRoles(superUsers);

  config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
  config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
  config.setWebServicePort(brokerWebServicePort);
  config.setBrokerServicePort(brokerServicePort);

  config.setAuthenticationEnabled(false);
  config.setTlsEnabled(false);
  config.setTlsAllowInsecureConnection(true);
  config.setAdvertisedAddress("localhost");

  final WorkerService functionsWorkerService = createPulsarFunctionWorker(config);
  final URL urlTls = new URL(brokerServiceUrl);
  final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
  try (final PulsarService pulsar = new PulsarService(config, functionWorkerService)) {
    pulsar.start();
    try (final PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).allowTlsInsecureConnection(true).build()) {
      // update cluster metadata
      final ClusterData clusterData = new ClusterData(urlTls.toString());
      admin.clusters().updateCluster(config.getClusterName(), clusterData);

      final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(workerConfig.getPulsarServiceUrl());
      try (final PulsarClient pulsarClient = clientBuilder.build()) {
        final TenantInfo propAdmin = new TenantInfo();
        propAdmin.getAdminRoles().add("superUser");
        propAdmin.setAllowedClusters(Sets.newHashSet(CLUSTER_NAME));
        admin.tenants().updateTenant(tenant, propAdmin);

        testPulsarFunction(admin, pulsarClient);
      }
    }
  }
}
 
Example 20
Source File: PulsarFunctionsITest.java    From java-specialagent with Apache License 2.0 4 votes vote down vote up
static void start() throws Exception {
  // Start local bookkeeper ensemble
  final LocalBookkeeperEnsemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT,TestUtil::nextFreePort);
  bkEnsemble.start();

  final String brokerServiceUrl = "http://127.0.0.1:" + brokerWebServicePort;

  final ServiceConfiguration config = spy(new ServiceConfiguration());
  config.setClusterName(CLUSTER_NAME);
  final Set<String> superUsers = Sets.newHashSet("superUser");
  config.setSuperUserRoles(superUsers);
  config.setWebServicePort(Optional.of(brokerWebServicePort));
  config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
  config.setBrokerServicePort(Optional.of(brokerServicePort));
  config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
  config.setTlsAllowInsecureConnection(true);
  config.setAdvertisedAddress("localhost");

  config.setAuthenticationEnabled(false);
  config.setAuthorizationEnabled(false);

  config.setBrokerClientTlsEnabled(false);
  config.setAllowAutoTopicCreationType("non-partitioned");

  final WorkerService functionsWorkerService = createPulsarFunctionWorker(config);
  final URL urlTls = new URL(brokerServiceUrl);
  final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService);
  try (final PulsarService pulsar = new PulsarService(config, functionWorkerService)) {
    pulsar.start();
    try (final PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).allowTlsInsecureConnection(true).build()) {
      // update cluster metadata
      final ClusterData clusterData = new ClusterData(urlTls.toString());
      admin.clusters().updateCluster(config.getClusterName(), clusterData);

      final TenantInfo propAdmin = new TenantInfo();
      propAdmin.getAdminRoles().add("superUser");
      propAdmin.setAllowedClusters(Sets.newHashSet(CLUSTER_NAME));
      admin.tenants().updateTenant(tenant, propAdmin);

      final String jarFilePathUrl = Utils.FILE + ":" + ExclamationFunction.class.getProtectionDomain().getCodeSource().getLocation().getPath();

      final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(workerConfig.getPulsarServiceUrl());
      try (final PulsarClient pulsarClient = clientBuilder.build()) {
        testE2EPulsarFunction(jarFilePathUrl, admin, pulsarClient);
      }
    }
  }
}