org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble Java Examples
The following examples show how to use
org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AdvertisedAddressTest.java From pulsar with Apache License 2.0 | 6 votes |
@BeforeMethod public void setup() throws Exception { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); ServiceConfiguration config = new ServiceConfiguration(); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setWebServicePort(Optional.ofNullable(0)); config.setClusterName("usc"); config.setAdvertisedAddress("localhost"); config.setBrokerServicePort(Optional.ofNullable(0)); config.setAdvertisedAddress(advertisedAddress); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); pulsar = new PulsarService(config); pulsar.start(); }
Example #2
Source File: PulsarClientTest.java From java-specialagent with Apache License 2.0 | 5 votes |
@BeforeClass public static void beforeClass() throws Exception { if (!isJdkSupported) return; bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, TestUtil::nextFreePort); bkEnsemble.start(); final int brokerWebServicePort = TestUtil.nextFreePort(); final int brokerServicePort = TestUtil.nextFreePort(); final ServiceConfiguration config = new ServiceConfiguration(); config.setClusterName(CLUSTER_NAME); final Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setWebServicePort(brokerWebServicePort); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setBrokerServicePort(brokerServicePort); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setTlsAllowInsecureConnection(true); config.setAdvertisedAddress("localhost"); pulsarService = new PulsarService(config); pulsarService.start(); try (final PulsarAdmin admin = pulsarService.getAdminClient()) { final ClusterData clusterData = new ClusterData(pulsarService.getBrokerServiceUrl()); admin.clusters().createCluster(CLUSTER_NAME, clusterData); final TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList(CLUSTER_NAME))); admin.tenants().createTenant("public", propAdmin); admin.namespaces().createNamespace("public/default", Sets.newHashSet(CLUSTER_NAME)); } }
Example #3
Source File: ZooKeeperClientAspectJTest.java From pulsar with Apache License 2.0 | 5 votes |
public void start() throws IOException { try { zks = new ZooKeeperServer(zkTmpDir, zkTmpDir, ZooKeeperServer.DEFAULT_TICK_TIME); zks.setMaxSessionTimeout(20000); serverFactory = new NIOServerCnxnFactory(); serverFactory.configure(new InetSocketAddress(zkPort), 1000); serverFactory.startup(zks); } catch (Exception e) { log.error("Exception while instantiating ZooKeeper", e); } LocalBookkeeperEnsemble.waitForServerUp(hostPort, 30000); log.info("ZooKeeper started at {}", hostPort); }
Example #4
Source File: SLAMonitoringTest.java From pulsar with Apache License 2.0 | 5 votes |
@BeforeClass void setup() throws Exception { log.info("---- Initializing SLAMonitoringTest -----"); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); // start brokers for (int i = 0; i < BROKER_COUNT; i++) { ServiceConfiguration config = new ServiceConfiguration(); config.setBrokerServicePort(Optional.of(0)); config.setClusterName("my-cluster"); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setBrokerServicePort(Optional.of(0)); config.setDefaultNumberOfNamespaceBundles(1); config.setLoadBalancerEnabled(false); configurations[i] = config; pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); brokerWebServicePorts[i] = pulsarServices[i].getListenPortHTTP().get(); brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get(); brokerUrls[i] = new URL(pulsarServices[i].getWebServiceAddress()); pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build(); } Thread.sleep(100); createTenant(pulsarAdmins[BROKER_COUNT - 1]); for (int i = 0; i < BROKER_COUNT; i++) { String topic = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]); pulsarAdmins[0].namespaces().createNamespace(topic); } }
Example #5
Source File: TopicOwnerTest.java From pulsar with Apache License 2.0 | 5 votes |
@BeforeMethod void setup() throws Exception { log.info("---- Initializing TopicOwnerTest -----"); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); // start brokers for (int i = 0; i < BROKER_COUNT; i++) { ServiceConfiguration config = new ServiceConfiguration(); config.setBrokerServicePort(Optional.of(0)); config.setClusterName("my-cluster"); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setDefaultNumberOfNamespaceBundles(1); config.setLoadBalancerEnabled(false); configurations[i] = config; pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); pulsarAdmins[i] = PulsarAdmin.builder() .serviceHttpUrl(pulsarServices[i].getWebServiceAddress()) .build(); } Thread.sleep(1000); }
Example #6
Source File: MaxMessageSizeTest.java From pulsar with Apache License 2.0 | 5 votes |
@BeforeMethod void setup() { try { bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); ServerConfiguration conf = new ServerConfiguration(); conf.setNettyMaxFrameSizeBytes(10 * 1024 * 1024); bkEnsemble.startStandalone(conf, false); configuration = new ServiceConfiguration(); configuration.setZookeeperServers("127.0.0.1:" + bkEnsemble.getZookeeperPort()); configuration.setAdvertisedAddress("localhost"); configuration.setWebServicePort(Optional.of(0)); configuration.setClusterName("max_message_test"); configuration.setBrokerServicePort(Optional.of(0)); configuration.setAuthorizationEnabled(false); configuration.setAuthenticationEnabled(false); configuration.setManagedLedgerMaxEntriesPerLedger(5); configuration.setManagedLedgerMinLedgerRolloverTimeMinutes(0); configuration.setMaxMessageSize(10 * 1024 * 1024); pulsar = new PulsarService(configuration); pulsar.start(); String url = "http://127.0.0.1:" + pulsar.getListenPortHTTP().get(); admin = PulsarAdmin.builder().serviceHttpUrl(url).build(); admin.clusters().createCluster("max_message_test", new ClusterData(url)); admin.tenants() .createTenant("test", new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("max_message_test"))); admin.namespaces().createNamespace("test/message", Sets.newHashSet("max_message_test")); } catch (Exception e) { e.printStackTrace(); } }
Example #7
Source File: BkEnsemblesTestBase.java From pulsar with Apache License 2.0 | 5 votes |
@BeforeMethod protected void setup() throws Exception { try { // start local bookie and zookeeper bkEnsemble = new LocalBookkeeperEnsemble(numberOfBookies, 0, () -> 0); bkEnsemble.start(); // start pulsar service config = new ServiceConfiguration(); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); config.setClusterName("usc"); config.setBrokerServicePort(Optional.of(0)); config.setAuthorizationEnabled(false); config.setAuthenticationEnabled(false); config.setManagedLedgerMaxEntriesPerLedger(5); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); config.setAdvertisedAddress("127.0.0.1"); config.setAllowAutoTopicCreationType("non-partitioned"); pulsar = new PulsarService(config); pulsar.start(); admin = PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()).build(); admin.clusters().createCluster("usc", new ClusterData(pulsar.getWebServiceAddress())); admin.tenants().createTenant("prop", new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("usc"))); } catch (Throwable t) { log.error("Error setting up broker test", t); Assert.fail("Broker test setup failed"); } }
Example #8
Source File: SimpleLoadManagerImplTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup() throws Exception { // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); // Start broker 1 ServiceConfiguration config1 = spy(new ServiceConfiguration()); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config1.setBrokerServicePort(Optional.of(0)); config1.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config1.setBrokerServicePortTls(Optional.of(0)); config1.setWebServicePortTls(Optional.of(0)); config1.setAdvertisedAddress("localhost"); pulsar1 = new PulsarService(config1); pulsar1.start(); url1 = new URL(pulsar1.getWebServiceAddress()); admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); brokerStatsClient1 = admin1.brokerStats(); primaryHost = pulsar1.getWebServiceAddress(); // Start broker 2 ServiceConfiguration config2 = new ServiceConfiguration(); config2.setClusterName("use"); config2.setWebServicePort(Optional.of(0)); config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config2.setBrokerServicePort(Optional.of(0)); config2.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config2.setBrokerServicePortTls(Optional.of(0)); config2.setWebServicePortTls(Optional.of(0)); config2.setAdvertisedAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); url2 = new URL(pulsar2.getWebServiceAddress()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); brokerStatsClient2 = admin2.brokerStats(); secondaryHost = pulsar2.getWebServiceAddress(); Thread.sleep(100); }
Example #9
Source File: PulsarFunctionTlsTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup(Method method) throws Exception { log.info("--- Setting up method {} ---", method.getName()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); config = spy(new ServiceConfiguration()); config.setClusterName("use"); Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); Set<String> providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); config.setAuthenticationEnabled(true); config.setAuthorizationEnabled(true); config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); config.setTlsAllowInsecureConnection(true); config.setAdvertisedAddress("localhost"); functionsWorkerService = spy(createPulsarFunctionWorker(config)); AuthenticationService authenticationService = new AuthenticationService(config); AuthorizationService authorizationService = new AuthorizationService(config, mock(ConfigurationCacheService.class)); when(functionsWorkerService.getAuthenticationService()).thenReturn(authenticationService); when(functionsWorkerService.getAuthorizationService()).thenReturn(authorizationService); when(functionsWorkerService.isInitialized()).thenReturn(true); PulsarAdmin admin = mock(PulsarAdmin.class); Tenants tenants = mock(Tenants.class); when(admin.tenants()).thenReturn(tenants); when(functionsWorkerService.getBrokerAdmin()).thenReturn(admin); Set<String> admins = Sets.newHashSet("superUser"); TenantInfo tenantInfo = new TenantInfo(admins, null); when(tenants.getTenantInfo(any())).thenReturn(tenantInfo); Namespaces namespaces = mock(Namespaces.class); when(admin.namespaces()).thenReturn(namespaces); when(namespaces.getNamespaces(any())).thenReturn(namespaceList); // mock: once authentication passes, function should return response: function already exist FunctionMetaDataManager dataManager = mock(FunctionMetaDataManager.class); when(dataManager.containsFunction(any(), any(), any())).thenReturn(true); when(functionsWorkerService.getFunctionMetaDataManager()).thenReturn(dataManager); workerServer = new WorkerServer(functionsWorkerService); workerServer.start(); Thread.sleep(2000); String functionTlsUrl = String.format("https://%s:%s", functionsWorkerService.getWorkerConfig().getWorkerHostname(), workerServer.getListenPortHTTPS().get()); Map<String, String> authParams = new HashMap<>(); authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); functionAdmin = PulsarAdmin.builder().serviceHttpUrl(functionTlsUrl) .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH).allowTlsInsecureConnection(true) .authentication(authTls).build(); Thread.sleep(100); }
Example #10
Source File: PulsarFunctionAdminTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup(Method method) throws Exception { log.info("--- Setting up method {} ---", method.getName()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); config = spy(new ServiceConfiguration()); config.setClusterName("use"); Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setWebServicePort(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); Set<String> providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); config.setAuthenticationEnabled(true); config.setAuthenticationProviders(providers); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); config.setTlsAllowInsecureConnection(true); functionsWorkerService = createPulsarFunctionWorker(config); Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService); pulsar = new PulsarService(config, functionWorkerService, (exitCode) -> {}); pulsar.start(); urlTls = new URL(pulsar.getBrokerServiceUrlTls()); Map<String, String> authParams = new HashMap<>(); authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); admin = spy( PulsarAdmin.builder() .serviceHttpUrl(pulsar.getWebServiceAddressTls()) .tlsTrustCertsFilePath(TLS_CLIENT_CERT_FILE_PATH) .allowTlsInsecureConnection(true) .authentication(authTls) .build()); brokerStatsClient = admin.brokerStats(); primaryHost = pulsar.getWebServiceAddress(); // update cluster metadata ClusterData clusterData = new ClusterData(urlTls.toString()); admin.clusters().updateCluster(config.getClusterName(), clusterData); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl()); if (isNotBlank(workerConfig.getClientAuthenticationPlugin()) && isNotBlank(workerConfig.getClientAuthenticationParameters())) { clientBuilder.enableTls(workerConfig.isUseTls()); clientBuilder.allowTlsInsecureConnection(workerConfig.isTlsAllowInsecureConnection()); clientBuilder.authentication(workerConfig.getClientAuthenticationPlugin(), workerConfig.getClientAuthenticationParameters()); } pulsarClient = clientBuilder.build(); TenantInfo propAdmin = new TenantInfo(); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().updateTenant(tenant, propAdmin); Thread.sleep(100); }
Example #11
Source File: PulsarWorkerAssignmentTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod(timeOut = 60000) void setup(Method method) throws Exception { log.info("--- Setting up method {} ---", method.getName()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); config = spy(new ServiceConfiguration()); config.setClusterName("use"); final Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setWebServicePort(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setBrokerServicePort(Optional.of(0)); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setAdvertisedAddress("localhost"); functionsWorkerService = createPulsarFunctionWorker(config); final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService); pulsar = new PulsarService(config, functionWorkerService, (exitCode) -> {}); pulsar.start(); admin = spy(PulsarAdmin.builder().serviceHttpUrl(pulsar.getWebServiceAddress()).build()); brokerStatsClient = admin.brokerStats(); primaryHost = pulsar.getWebServiceAddress(); // update cluster metadata final ClusterData clusterData = new ClusterData(pulsar.getBrokerServiceUrl()); admin.clusters().updateCluster(config.getClusterName(), clusterData); final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl()); pulsarClient = clientBuilder.build(); final TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().updateTenant(tenant, propAdmin); Thread.sleep(100); }
Example #12
Source File: PulsarFunctionPublishTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup(Method method) throws Exception { // delete all function temp files File dir = new File(System.getProperty("java.io.tmpdir")); File[] foundFiles = dir.listFiles((ignoredDir, name) -> name.startsWith("function")); for (File file : foundFiles) { file.delete(); } log.info("--- Setting up method {} ---", method.getName()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); config = spy(new ServiceConfiguration()); config.setClusterName("use"); Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setWebServicePort(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setBrokerServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setTlsAllowInsecureConnection(true); config.setAdvertisedAddress("localhost"); Set<String> providers = new HashSet<>(); providers.add(AuthenticationProviderTls.class.getName()); config.setAuthenticationEnabled(true); config.setAuthenticationProviders(providers); config.setAuthorizationEnabled(true); config.setAuthorizationProvider(PulsarAuthorizationProvider.class.getName()); config.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH); config.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH); config.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); config.setBrokerClientAuthenticationPlugin(AuthenticationTls.class.getName()); config.setBrokerClientAuthenticationParameters( "tlsCertFile:" + TLS_CLIENT_CERT_FILE_PATH + "," + "tlsKeyFile:" + TLS_CLIENT_KEY_FILE_PATH); config.setBrokerClientTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH); config.setBrokerClientTlsEnabled(true); config.setAllowAutoTopicCreationType("non-partitioned"); functionsWorkerService = createPulsarFunctionWorker(config); Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService); pulsar = new PulsarService(config, functionWorkerService, (exitCode) -> {}); pulsar.start(); String brokerServiceUrl = pulsar.getWebServiceAddressTls(); urlTls = new URL(brokerServiceUrl); Map<String, String> authParams = new HashMap<>(); authParams.put("tlsCertFile", TLS_CLIENT_CERT_FILE_PATH); authParams.put("tlsKeyFile", TLS_CLIENT_KEY_FILE_PATH); Authentication authTls = new AuthenticationTls(); authTls.configure(authParams); admin = spy( PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).tlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH) .allowTlsInsecureConnection(true).authentication(authTls).build()); brokerStatsClient = admin.brokerStats(); primaryHost = pulsar.getWebServiceAddress(); // update cluster metadata ClusterData clusterData = new ClusterData(urlTls.toString()); admin.clusters().updateCluster(config.getClusterName(), clusterData); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(this.workerConfig.getPulsarServiceUrl()); if (isNotBlank(workerConfig.getClientAuthenticationPlugin()) && isNotBlank(workerConfig.getClientAuthenticationParameters())) { clientBuilder.enableTls(workerConfig.isUseTls()); clientBuilder.allowTlsInsecureConnection(workerConfig.isTlsAllowInsecureConnection()); clientBuilder.authentication(workerConfig.getClientAuthenticationPlugin(), workerConfig.getClientAuthenticationParameters()); } pulsarClient = clientBuilder.build(); TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().updateTenant(tenant, propAdmin); System.setProperty(JAVA_INSTANCE_JAR_PROPERTY, FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath()); functionWorkerService.get().getLeaderService().waitLeaderInit(); }
Example #13
Source File: BrokerBookieIsolationTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod protected void setup() throws Exception { // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(4, 0, () -> 0); bkEnsemble.start(); }
Example #14
Source File: BacklogQuotaManagerTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup() throws Exception { try { // start local bookie and zookeeper bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); // start pulsar service config = new ServiceConfiguration(); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.ofNullable(0)); config.setClusterName("usc"); config.setBrokerServicePort(Optional.ofNullable(0)); config.setAuthorizationEnabled(false); config.setAuthenticationEnabled(false); config.setBacklogQuotaCheckIntervalInSeconds(TIME_TO_CHECK_BACKLOG_QUOTA); config.setManagedLedgerMaxEntriesPerLedger(MAX_ENTRIES_PER_LEDGER); config.setManagedLedgerMinLedgerRolloverTimeMinutes(0); config.setAllowAutoTopicCreationType("non-partitioned"); pulsar = new PulsarService(config); pulsar.start(); adminUrl = new URL("http://127.0.0.1" + ":" + pulsar.getListenPortHTTP().get()); admin = PulsarAdmin.builder().serviceHttpUrl(adminUrl.toString()).build(); admin.clusters().createCluster("usc", new ClusterData(adminUrl.toString())); admin.tenants().createTenant("prop", new TenantInfo(Sets.newHashSet("appid1"), Sets.newHashSet("usc"))); admin.namespaces().createNamespace("prop/ns-quota"); admin.namespaces().setNamespaceReplicationClusters("prop/ns-quota", Sets.newHashSet("usc")); admin.namespaces().createNamespace("prop/quotahold"); admin.namespaces().setNamespaceReplicationClusters("prop/quotahold", Sets.newHashSet("usc")); admin.namespaces().createNamespace("prop/quotaholdasync"); admin.namespaces().setNamespaceReplicationClusters("prop/quotaholdasync", Sets.newHashSet("usc")); } catch (Throwable t) { LOG.error("Error setting up broker test", t); fail("Broker test setup failed"); } }
Example #15
Source File: KafkaStandalone.java From kop with Apache License 2.0 | 4 votes |
public void setBkEnsemble(LocalBookkeeperEnsemble bkEnsemble) { this.bkEnsemble = bkEnsemble; }
Example #16
Source File: LoadBalancerTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup() throws Exception { // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); ZkUtils.createFullPathOptimistic(bkEnsemble.getZkClient(), SimpleLoadManagerImpl.LOADBALANCER_DYNAMIC_SETTING_STRATEGY_ZPATH, "{\"loadBalancerStrategy\":\"leastLoadedServer\"}".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT); final String localhost = "localhost"; // start brokers for (int i = 0; i < BROKER_COUNT; i++) { ServiceConfiguration config = new ServiceConfiguration(); config.setBrokerServicePort(Optional.ofNullable(brokerNativeBrokerPorts[i])); config.setClusterName("use"); config.setAdvertisedAddress(localhost); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); config.setBrokerServicePortTls(Optional.of(0)); config.setWebServicePortTls(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setBrokerServicePort(Optional.of(0)); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setAdvertisedAddress(localhost+i); config.setLoadBalancerEnabled(false); pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); brokerWebServicePorts[i] = pulsarServices[i].getListenPortHTTP().get(); brokerNativeBrokerPorts[i] = pulsarServices[i].getBrokerListenPort().get(); brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]); lookupAddresses[i] = pulsarServices[i].getAdvertisedAddress() + ":" + pulsarServices[i].getListenPortHTTP().get(); pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build(); } createNamespacePolicies(pulsarServices[0]); Thread.sleep(100); }
Example #17
Source File: ModularLoadManagerImplTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup() throws Exception { executor = new ThreadPoolExecutor(1, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<>()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); // Start broker 1 ServiceConfiguration config1 = new ServiceConfiguration(); config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config1.setAdvertisedAddress("localhost"); config1.setBrokerServicePort(Optional.of(0)); config1.setBrokerServicePortTls(Optional.of(0)); config1.setWebServicePortTls(Optional.of(0)); pulsar1 = new PulsarService(config1); pulsar1.start(); primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get()); url1 = new URL(pulsar1.getWebServiceAddress()); admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); // Start broker 2 ServiceConfiguration config2 = new ServiceConfiguration(); config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config2.setClusterName("use"); config2.setWebServicePort(Optional.of(0)); config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config2.setAdvertisedAddress("localhost"); config2.setBrokerServicePort(Optional.of(0)); config2.setBrokerServicePortTls(Optional.of(0)); config2.setWebServicePortTls(Optional.of(0)); pulsar2 = new PulsarService(config2); pulsar2.start(); secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); url2 = new URL(pulsar2.getWebServiceAddress()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager"); secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager"); nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32()); Thread.sleep(100); }
Example #18
Source File: AntiAffinityNamespaceGroupTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod void setup() throws Exception { executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); // Start broker 1 ServiceConfiguration config1 = new ServiceConfiguration(); config1.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config1.setClusterName("use"); config1.setWebServicePort(Optional.of(0)); config1.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config1.setBrokerServicePort(Optional.of(0)); config1.setFailureDomainsEnabled(true); config1.setLoadBalancerEnabled(true); config1.setAdvertisedAddress("localhost"); createCluster(bkEnsemble.getZkClient(), config1); pulsar1 = new PulsarService(config1); pulsar1.start(); primaryHost = String.format("%s:%d", "localhost", pulsar1.getListenPortHTTP().get()); url1 = new URL("http://127.0.0.1" + ":" + pulsar1.getListenPortHTTP().get()); admin1 = PulsarAdmin.builder().serviceHttpUrl(url1.toString()).build(); // Start broker 2 ServiceConfiguration config2 = new ServiceConfiguration(); config2.setLoadManagerClassName(ModularLoadManagerImpl.class.getName()); config2.setClusterName("use"); config2.setWebServicePort(Optional.of(0)); config2.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config2.setBrokerServicePort(Optional.of(0)); config2.setFailureDomainsEnabled(true); config2.setAdvertisedAddress("localhost"); pulsar2 = new PulsarService(config2); pulsar2.start(); secondaryHost = String.format("%s:%d", "localhost", pulsar2.getListenPortHTTP().get()); url2 = new URL("http://127.0.0.1" + ":" + config2.getWebServicePort().get()); admin2 = PulsarAdmin.builder().serviceHttpUrl(url2.toString()).build(); primaryLoadManager = (ModularLoadManagerImpl) getField(pulsar1.getLoadManager().get(), "loadManager"); secondaryLoadManager = (ModularLoadManagerImpl) getField(pulsar2.getLoadManager().get(), "loadManager"); nsFactory = new NamespaceBundleFactory(pulsar1, Hashing.crc32()); Thread.sleep(100); }
Example #19
Source File: TransactionMetaStoreTestBase.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeClass void setup() throws Exception { log.info("---- Initializing SLAMonitoringTest -----"); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); String[] args = new String[]{ "--cluster", "my-cluster", "--configuration-store", "localhost:" + bkEnsemble.getZookeeperPort(), "--initial-num-transaction-coordinators", "16"}; PulsarTransactionCoordinatorMetadataSetup.main(args); // start brokers for (int i = 0; i < BROKER_COUNT; i++) { ServiceConfiguration config = new ServiceConfiguration(); config.setBrokerServicePort(Optional.of(0)); config.setClusterName("my-cluster"); config.setAdvertisedAddress("localhost"); config.setWebServicePort(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setDefaultNumberOfNamespaceBundles(1); config.setLoadBalancerEnabled(false); configurations[i] = config; pulsarServices[i] = new PulsarService(config); pulsarServices[i].start(); pulsarAdmins[i] = PulsarAdmin.builder() .serviceHttpUrl(pulsarServices[i].getWebServiceAddress()) .build(); } Thread.sleep(100); PulsarClient client = PulsarClient.builder(). serviceUrl(pulsarServices[0].getBrokerServiceUrl()) .build(); transactionCoordinatorClient = new TransactionCoordinatorClientImpl(client); transactionCoordinatorClient.start(); Thread.sleep(3000); }
Example #20
Source File: ClientDeduplicationFailureTest.java From pulsar with Apache License 2.0 | 4 votes |
@BeforeMethod(timeOut = 300000) void setup(Method method) throws Exception { log.info("--- Setting up method {} ---", method.getName()); // Start local bookkeeper ensemble bkEnsemble = new LocalBookkeeperEnsemble(3, 0, () -> 0); bkEnsemble.start(); config = spy(new ServiceConfiguration()); config.setClusterName("use"); config.setWebServicePort(Optional.of(0)); config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort()); config.setBrokerServicePort(Optional.of(0)); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setTlsAllowInsecureConnection(true); config.setAdvertisedAddress("localhost"); config.setLoadBalancerSheddingEnabled(false); config.setLoadBalancerAutoBundleSplitEnabled(false); config.setLoadBalancerEnabled(false); config.setLoadBalancerAutoUnloadSplitBundlesEnabled(false); config.setAllowAutoTopicCreationType("non-partitioned"); pulsar = new PulsarService(config); pulsar.start(); String brokerServiceUrl = pulsar.getWebServiceAddress(); url = new URL(brokerServiceUrl); admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).build(); brokerStatsClient = admin.brokerStats(); primaryHost = pulsar.getWebServiceAddress(); // update cluster metadata ClusterData clusterData = new ClusterData(url.toString()); admin.clusters().createCluster(config.getClusterName(), clusterData); ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(pulsar.getBrokerServiceUrl()).maxBackoffInterval(1, TimeUnit.SECONDS); pulsarClient = clientBuilder.build(); TenantInfo tenantInfo = new TenantInfo(); tenantInfo.setAllowedClusters(Sets.newHashSet(Lists.newArrayList("use"))); admin.tenants().createTenant(tenant, tenantInfo); }
Example #21
Source File: PulsarStandalone.java From pulsar with Apache License 2.0 | 4 votes |
public void setBkEnsemble(LocalBookkeeperEnsemble bkEnsemble) { this.bkEnsemble = bkEnsemble; }
Example #22
Source File: PulsarFunctionsITest.java From java-specialagent with Apache License 2.0 | 4 votes |
static void start() throws Exception { // Start local bookkeeper ensemble final LocalBookkeeperEnsemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, TestUtil::nextFreePort); bkEnsemble.start(); final String brokerServiceUrl = "http://127.0.0.1:" + brokerWebServicePort; final ServiceConfiguration config = new ServiceConfiguration(); config.setClusterName(CLUSTER_NAME); final Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setWebServicePort(brokerWebServicePort); config.setBrokerServicePort(brokerServicePort); config.setAuthenticationEnabled(false); config.setTlsEnabled(false); config.setTlsAllowInsecureConnection(true); config.setAdvertisedAddress("localhost"); final WorkerService functionsWorkerService = createPulsarFunctionWorker(config); final URL urlTls = new URL(brokerServiceUrl); final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService); try (final PulsarService pulsar = new PulsarService(config, functionWorkerService)) { pulsar.start(); try (final PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).allowTlsInsecureConnection(true).build()) { // update cluster metadata final ClusterData clusterData = new ClusterData(urlTls.toString()); admin.clusters().updateCluster(config.getClusterName(), clusterData); final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(workerConfig.getPulsarServiceUrl()); try (final PulsarClient pulsarClient = clientBuilder.build()) { final TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(CLUSTER_NAME)); admin.tenants().updateTenant(tenant, propAdmin); testPulsarFunction(admin, pulsarClient); } } } }
Example #23
Source File: PulsarFunctionsITest.java From java-specialagent with Apache License 2.0 | 4 votes |
static void start() throws Exception { // Start local bookkeeper ensemble final LocalBookkeeperEnsemble bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT,TestUtil::nextFreePort); bkEnsemble.start(); final String brokerServiceUrl = "http://127.0.0.1:" + brokerWebServicePort; final ServiceConfiguration config = spy(new ServiceConfiguration()); config.setClusterName(CLUSTER_NAME); final Set<String> superUsers = Sets.newHashSet("superUser"); config.setSuperUserRoles(superUsers); config.setWebServicePort(Optional.of(brokerWebServicePort)); config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT); config.setBrokerServicePort(Optional.of(brokerServicePort)); config.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName()); config.setTlsAllowInsecureConnection(true); config.setAdvertisedAddress("localhost"); config.setAuthenticationEnabled(false); config.setAuthorizationEnabled(false); config.setBrokerClientTlsEnabled(false); config.setAllowAutoTopicCreationType("non-partitioned"); final WorkerService functionsWorkerService = createPulsarFunctionWorker(config); final URL urlTls = new URL(brokerServiceUrl); final Optional<WorkerService> functionWorkerService = Optional.of(functionsWorkerService); try (final PulsarService pulsar = new PulsarService(config, functionWorkerService)) { pulsar.start(); try (final PulsarAdmin admin = PulsarAdmin.builder().serviceHttpUrl(brokerServiceUrl).allowTlsInsecureConnection(true).build()) { // update cluster metadata final ClusterData clusterData = new ClusterData(urlTls.toString()); admin.clusters().updateCluster(config.getClusterName(), clusterData); final TenantInfo propAdmin = new TenantInfo(); propAdmin.getAdminRoles().add("superUser"); propAdmin.setAllowedClusters(Sets.newHashSet(CLUSTER_NAME)); admin.tenants().updateTenant(tenant, propAdmin); final String jarFilePathUrl = Utils.FILE + ":" + ExclamationFunction.class.getProtectionDomain().getCodeSource().getLocation().getPath(); final ClientBuilder clientBuilder = PulsarClient.builder().serviceUrl(workerConfig.getPulsarServiceUrl()); try (final PulsarClient pulsarClient = clientBuilder.build()) { testE2EPulsarFunction(jarFilePathUrl, admin, pulsarClient); } } } }
Example #24
Source File: KafkaStandalone.java From kop with Apache License 2.0 | 4 votes |
public void start() throws Exception { if (config == null) { throw new IllegalArgumentException("Null configuration is provided"); } if (config.getAdvertisedAddress() != null && !config.getListeners().contains(config.getAdvertisedAddress())) { String err = "Error config: advertisedAddress - " + config.getAdvertisedAddress() + " and listeners - " + config.getListeners() + " not match."; log.error(err); throw new IllegalArgumentException(err); } log.info("--- setup KafkaStandaloneStarter ---"); if (!this.isOnlyBroker()) { ServerConfiguration bkServerConf = new ServerConfiguration(); bkServerConf.loadConf(new File(configFile).toURI().toURL()); // Start LocalBookKeeper bkEnsemble = new LocalBookkeeperEnsemble( this.getNumOfBk(), this.getZkPort(), this.getBkPort(), this.getStreamStoragePort(), this.getZkDir(), this.getBkDir(), this.isWipeData(), "127.0.0.1"); bkEnsemble.startStandalone(bkServerConf, !this.isNoStreamStorage()); } if (this.isNoBroker()) { return; } // Start Broker kafkaBroker = new KafkaService(config); kafkaBroker.start(); URL webServiceUrl = new URL( String.format("http://%s:%d", config.getAdvertisedAddress(), config.getWebServicePort().get())); final String brokerServiceUrl = String.format("pulsar://%s:%d", config.getAdvertisedAddress(), config.getBrokerServicePort().get()); admin = PulsarAdmin.builder().serviceHttpUrl(webServiceUrl.toString()).authentication( config.getBrokerClientAuthenticationPlugin(), config.getBrokerClientAuthenticationParameters()).build(); final String cluster = config.getClusterName(); createDefaultNameSpace(webServiceUrl, brokerServiceUrl, cluster); log.info("--- setup completed ---"); }