org.apache.pulsar.broker.ServiceConfiguration Java Examples

private void buildConf(ServiceConfiguration conf) {
    conf.setLoadBalancerEnabled(true);
    conf.setTlsCertificateFilePath(getTLSFile("broker.cert"));
    conf.setTlsKeyFilePath(getTLSFile("broker.key-pk8"));
    conf.setTlsTrustCertsFilePath(getTLSFile("ca.cert"));
    conf.setAuthenticationEnabled(true);
    conf.setSuperUserRoles(ImmutableSet.of("superproxy", "broker.pulsar.apache.org"));
    conf.setAuthenticationProviders(
            ImmutableSet.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls"));
    conf.setAuthorizationEnabled(true);
    conf.setBrokerClientTlsEnabled(true);
    String str = String.format("tlsCertFile:%s,tlsKeyFile:%s", getTLSFile("broker.cert"), getTLSFile("broker.key-pk8"));
    conf.setBrokerClientAuthenticationParameters(str);
    conf.setBrokerClientAuthenticationPlugin("org.apache.pulsar.client.impl.auth.AuthenticationTls");
    conf.setBrokerClientTrustCertsFilePath(getTLSFile("ca.cert"));
    conf.setTlsAllowInsecureConnection(true);
}
 

@Override
public void initialize(ServiceConfiguration config) throws IOException {
    File confFile = new File(System.getProperty(CONF_SYSTEM_PROPERTY_KEY));
    if (!confFile.exists()) {
        throw new IOException("The password auth conf file does not exist");
    } else if (!confFile.isFile()) {
        throw new IOException("The path is not a file");
    }

    @Cleanup BufferedReader reader = new BufferedReader(new FileReader(confFile));
    users = new HashMap<>();
    for (String line : reader.lines().toArray(s -> new String[s])) {
        List<String> splitLine = Arrays.asList(line.split(":"));
        if (splitLine.size() != 2) {
            throw new IOException("The format of the password auth conf file is invalid");
        }
        users.put(splitLine.get(0), splitLine.get(1));
    }
}
 

public static DelayedDeliveryTrackerFactory loadDelayedDeliveryTrackerFactory(ServiceConfiguration conf)
        throws IOException {
    Class<?> factoryClass;
    try {
        factoryClass = Class.forName(conf.getDelayedDeliveryTrackerFactoryClassName());
        Object obj = factoryClass.newInstance();
        checkArgument(obj instanceof DelayedDeliveryTrackerFactory,
                "The factory has to be an instance of " + DelayedDeliveryTrackerFactory.class.getName());

        DelayedDeliveryTrackerFactory factory = (DelayedDeliveryTrackerFactory) obj;
        factory.initialize(conf);
        return factory;
    } catch (Exception e) {
        throw new IOException(e);
    }
}
 

@Test(expectedExceptions = AuthenticationException.class)
public void testAuthenticateWhenInvalidTokenIsPassed() throws AuthenticationException, IOException {
    SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256);

    Properties properties = new Properties();
    properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY,
            AuthTokenUtils.encodeKeyBase64(secretKey));

    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setProperties(properties);

    AuthenticationProviderToken provider = new AuthenticationProviderToken();
    provider.initialize(conf);
    provider.authenticate(new AuthenticationDataSource() {
        @Override
        public String getHttpHeader(String name) {
            return AuthenticationProviderToken.HTTP_HEADER_VALUE_PREFIX + "invalid_token";
        }

        @Override
        public boolean hasDataFromHttp() {
            return true;
        }
    });
}
 

@BeforeClass
public void setup() throws Exception {

    // Set provider domain name
    properties = new Properties();
    properties.setProperty("athenzDomainNames", "test_provider");
    config = new ServiceConfiguration();
    config.setProperties(properties);

    // Initialize authentication provider
    provider = new AuthenticationProviderAthenz();
    provider.initialize(config);

    // Specify Athenz configuration file for AuthZpeClient which is used in AuthenticationProviderAthenz
    System.setProperty(ZpeConsts.ZPE_PROP_ATHENZ_CONF, "./src/test/resources/athenz.conf.test");
}
 

@BeforeMethod
public void setup() {
    clock = mock(Clock.class);
    when(clock.millis()).thenAnswer(invocation -> currentTime);

    conf = new ServiceConfiguration();
    conf.setReplicatedSubscriptionsSnapshotTimeoutSeconds(3);

    markers = new ArrayList<>();

    controller = mock(ReplicatedSubscriptionsController.class);
    when(controller.localCluster()).thenReturn(localCluster);
    doAnswer(invocation -> {
        ByteBuf marker = invocation.getArgument(0, ByteBuf.class);
        Commands.skipMessageMetadata(marker);
        markers.add(marker);
        return null;
    }).when(controller)
            .writeMarker(any(ByteBuf.class));
}
 

@Test
public void testLeastLongTermMessageRate() {
    BundleData bundleData = new BundleData();
    BrokerData brokerData1 = initBrokerData();
    BrokerData brokerData2 = initBrokerData();
    BrokerData brokerData3 = initBrokerData();
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(100);
    brokerData2.getTimeAverageData().setLongTermMsgRateIn(200);
    brokerData3.getTimeAverageData().setLongTermMsgRateIn(300);
    LoadData loadData = new LoadData();
    Map<String, BrokerData> brokerDataMap = loadData.getBrokerData();
    brokerDataMap.put("1", brokerData1);
    brokerDataMap.put("2", brokerData2);
    brokerDataMap.put("3", brokerData3);
    ServiceConfiguration conf = new ServiceConfiguration();
    ModularLoadManagerStrategy strategy = new LeastLongTermMessageRate(conf);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("1"));
    brokerData1.getTimeAverageData().setLongTermMsgRateIn(400);
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("2"));
    brokerData2.getLocalData().setCpu(new ResourceUsage(90, 100));
    assertEquals(strategy.selectBroker(brokerDataMap.keySet(), bundleData, loadData, conf), Optional.of("3"));
}
 

private static double getScore(final BrokerData brokerData, final ServiceConfiguration conf) {
    final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0;
    final double maxUsage = brokerData.getLocalData().getMaxResourceUsage();
    if (maxUsage > overloadThreshold) {
        log.warn("Broker {} is overloaded: max usage={}", brokerData.getLocalData().getWebServiceUrl(), maxUsage);
        return Double.POSITIVE_INFINITY;
    }

    double totalMessageRate = 0;
    for (BundleData bundleData : brokerData.getPreallocatedBundleData().values()) {
        final TimeAverageMessageData longTermData = bundleData.getLongTermData();
        totalMessageRate += longTermData.getMsgRateIn() + longTermData.getMsgRateOut();
    }

    // calculate estimated score
    final TimeAverageBrokerData timeAverageData = brokerData.getTimeAverageData();
    final double timeAverageLongTermMessageRate = timeAverageData.getLongTermMsgRateIn()
            + timeAverageData.getLongTermMsgRateOut();
    final double totalMessageRateEstimate = totalMessageRate + timeAverageLongTermMessageRate;

    if (log.isDebugEnabled()) {
        log.debug("Broker {} has long term message rate {}",
                brokerData.getLocalData().getWebServiceUrl(), totalMessageRateEstimate);
    }
    return totalMessageRateEstimate;
}
 

@Test
public void testConfigurationConverting() throws Exception {
    MockConfiguration mockConfiguration = new MockConfiguration();
    ServiceConfiguration serviceConfiguration = PulsarConfigurationLoader.convertFrom(mockConfiguration);

    // check whether converting correctly
    assertEquals(serviceConfiguration.getZookeeperServers(), "localhost:2181");
    assertEquals(serviceConfiguration.getConfigurationStoreServers(), "localhost:2184");
    assertEquals(serviceConfiguration.getBrokerServicePort().get(), new Integer(7650));
    assertEquals(serviceConfiguration.getBrokerServicePortTls().get(), new Integer(7651));
    assertEquals(serviceConfiguration.getWebServicePort().get(), new Integer(9080));
    assertEquals(serviceConfiguration.getWebServicePortTls().get(), new Integer(9443));

    // check whether exception causes
    try {
        PulsarConfigurationLoader.convertFrom(mockConfiguration, false);
        fail();
    } catch (Exception e) {
        assertEquals(e.getClass(), IllegalArgumentException.class);
    }
}
 

@Test
public void testConfigurationChangedByServiceConfiguration() throws Exception {
    File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
    if (testConfigFile.exists()) {
        testConfigFile.delete();
    }
    final String advertisedAddress1 = "advertisedAddress1";
    final String advertisedAddress2 = "advertisedAddress2";

    PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
    printWriter.println("advertisedAddress=" + advertisedAddress1);
    printWriter.close();
    testConfigFile.deleteOnExit();

    InputStream stream = new FileInputStream(testConfigFile);
    final ServiceConfiguration serviceConfiguration =
            ConfigurationUtils.create(stream, ServiceConfiguration.class);

    serviceConfiguration.setAdvertisedAddress(advertisedAddress2);

    final KafkaServiceConfiguration kafkaServiceConfig = ConfigurationUtils
            .create(serviceConfiguration.getProperties(), KafkaServiceConfiguration.class);

    assertEquals(kafkaServiceConfig.getAdvertisedAddress(), advertisedAddress1);
    assertEquals(serviceConfiguration.getAdvertisedAddress(), advertisedAddress2);
}
 

@Test
public void testMaxConsumersFailoverForBroker() throws Exception {
    // set max clients
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    doReturn(2).when(svcConfig).getMaxConsumersPerSubscription();
    doReturn(3).when(svcConfig).getMaxConsumersPerTopic();
    doReturn(svcConfig).when(pulsar).getConfiguration();

    testMaxConsumersFailover();
}
 

@Test(expectedExceptions = IOException.class)
public void testValidationKeyWhenBlankSecretKeyIsPassed() throws IOException {
    Properties properties = new Properties();
    properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "   ");

    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setProperties(properties);

    AuthenticationProviderToken provider = new AuthenticationProviderToken();
    provider.initialize(conf);
}
 

@Override
public BookKeeper create(ServiceConfiguration conf, ZooKeeper zkClient,
                         Optional<Class<? extends EnsemblePlacementPolicy>> ensemblePlacementPolicyClass,
                         Map<String, Object> properties, StatsLogger statsLogger) {
    // Always return the same instance (so that we don't loose the mock BK content on broker restart
    return mockBookKeeper;
}
 

public ReplicatedSubscriptionsSnapshotBuilder(ReplicatedSubscriptionsController controller,
        List<String> remoteClusters, ServiceConfiguration conf, Clock clock) {
    this.snapshotId = UUID.randomUUID().toString();
    this.controller = controller;
    this.remoteClusters = remoteClusters;
    this.missingClusters = new TreeSet<>(remoteClusters);
    this.clock = clock;
    this.timeoutMillis = TimeUnit.SECONDS.toMillis(conf.getReplicatedSubscriptionsSnapshotTimeoutSeconds());

    // If we have more than 2 cluster, we need to do 2 rounds of snapshots, to make sure
    // we're catching all the messages eventually exchanged between the two.
    this.needTwoRounds = remoteClusters.size() > 1;
}
 

private WorkerService createPulsarFunctionWorker(ServiceConfiguration config) {

        System.setProperty(JAVA_INSTANCE_JAR_PROPERTY,
                FutureUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath());

        workerConfig = new WorkerConfig();
        workerConfig.setPulsarFunctionsNamespace(pulsarFunctionsNamespace);
        workerConfig.setSchedulerClassName(
                org.apache.pulsar.functions.worker.scheduler.RoundRobinScheduler.class.getName());
        workerConfig.setFunctionRuntimeFactoryClassName(ThreadRuntimeFactory.class.getName());
        workerConfig.setFunctionRuntimeFactoryConfigs(
                ObjectMapperFactory.getThreadLocal().convertValue(new ThreadRuntimeFactoryConfig().setThreadGroupName("use"), Map.class));        // worker talks to local broker
        workerConfig.setFailureCheckFreqMs(100);
        workerConfig.setNumFunctionPackageReplicas(1);
        workerConfig.setClusterCoordinationTopicName("coordinate");
        workerConfig.setFunctionAssignmentTopicName("assignment");
        workerConfig.setFunctionMetadataTopicName("metadata");
        workerConfig.setInstanceLivenessCheckFreqMs(100);
        workerConfig.setWorkerPort(0);
        workerConfig.setPulsarFunctionsCluster(config.getClusterName());
        String hostname = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(config.getAdvertisedAddress());
        this.workerId = "c-" + config.getClusterName() + "-fw-" + hostname + "-" + workerConfig.getWorkerPort();
        workerConfig.setWorkerHostname(hostname);
        workerConfig.setWorkerId(workerId);

        workerConfig.setClientAuthenticationPlugin(AuthenticationTls.class.getName());
        workerConfig.setClientAuthenticationParameters(
                String.format("tlsCertFile:%s,tlsKeyFile:%s", TLS_CLIENT_CERT_FILE_PATH, TLS_CLIENT_KEY_FILE_PATH));
        workerConfig.setUseTls(true);
        workerConfig.setTlsAllowInsecureConnection(true);
        workerConfig.setTlsTrustCertsFilePath(TLS_TRUST_CERT_FILE_PATH);

        workerConfig.setAuthenticationEnabled(true);
        workerConfig.setAuthorizationEnabled(true);

        return new WorkerService(workerConfig);
    }
 

public WebSocketService(ClusterData localCluster, ServiceConfiguration config) {
    this.config = config;
    this.localCluster = localCluster;
    this.topicProducerMap = new ConcurrentOpenHashMap<>();
    this.topicConsumerMap = new ConcurrentOpenHashMap<>();
    this.topicReaderMap = new ConcurrentOpenHashMap<>();
    this.proxyStats = new ProxyStats(this);
}
 

@Test
public void testPulsarConfiguraitonComplete() throws Exception {
    final String zk = "localhost:2184";
    final Properties prop = new Properties();
    prop.setProperty("zookeeperServers", zk);
    final ServiceConfiguration serviceConfig = PulsarConfigurationLoader.create(prop, ServiceConfiguration.class);
    try {
        isComplete(serviceConfig);
        fail("it should fail as config is not complete");
    } catch (IllegalArgumentException e) {
        // Ok
    }
}
 

@Test(expectedExceptions = IOException.class)
public void testInitializeWhenSecretKeyIsValidPathOrBase64() throws IOException {
    Properties properties = new Properties();
    properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY,
            "secret_key_file_not_exist");

    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setProperties(properties);

    new AuthenticationProviderToken().initialize(conf);
}
 

private void validateConfigKey(String key) {
    try {
        ServiceConfiguration.class.getDeclaredField(key);
    } catch (Exception e) {
        log.error("ServiceConfiguration key {} not found {}", key, e.getMessage());
        throw new IllegalArgumentException("Invalid service config " + key, e);
    }
}
 

@Test
public void testMaxProducersForNamespace() throws Exception {
    ServiceConfiguration svcConfig = spy(new ServiceConfiguration());
    doReturn(svcConfig).when(pulsar).getConfiguration();
    // set max clients
    Policies policies = new Policies();
    policies.max_producers_per_topic = 2;
    when(pulsar.getConfigurationCache().policiesCache()
            .get(AdminResource.path(POLICIES, TopicName.get(successTopicName).getNamespace())))
            .thenReturn(Optional.of(policies));
    testMaxProducers();
}
 

@Test(expectedExceptions = IllegalArgumentException.class)
public void testValidationWhenPublicKeyAlgIsInvalid() throws IOException {
    Properties properties = new Properties();
    properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_PUBLIC_ALG,
            "invalid");

    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setProperties(properties);

    new AuthenticationProviderToken().initialize(conf);
}
 

private static void testTokenAudienceWithDifferentConfig(Properties properties,
                                                         String audienceClaim, List<String> audiences) throws Exception {
    @Cleanup
    AuthenticationProviderToken provider = new AuthenticationProviderToken();
    SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256);

    File secretKeyFile = File.createTempFile("pulsar-test-secret-key-valid", ".key");
    secretKeyFile.deleteOnExit();
    Files.write(Paths.get(secretKeyFile.toString()), secretKey.getEncoded());

    properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, secretKeyFile.toString());
    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setProperties(properties);
    provider.initialize(conf);

    String token = createTokenWithAudience(secretKey, audienceClaim, audiences);

    // Pulsar protocol auth
    String subject = provider.authenticate(new AuthenticationDataSource() {
        @Override
        public boolean hasDataFromCommand() {
            return true;
        }

        @Override
        public String getCommandData() {
            return token;
        }
    });
    assertEquals(subject, SUBJECT);
    provider.close();
}
 

@Test
public void testPulsarConfiguraitonLoadingProp() throws Exception {
    final String zk = "localhost:2184";
    final Properties prop = new Properties();
    prop.setProperty("zookeeperServers", zk);
    final ServiceConfiguration serviceConfig = PulsarConfigurationLoader.create(prop, ServiceConfiguration.class);
    assertNotNull(serviceConfig);
    assertEquals(serviceConfig.getZookeeperServers(), zk);
}
 

@Test
public void testAuthSecretKeyFromFile() throws Exception {
    SecretKey secretKey = AuthTokenUtils.createSecretKey(SignatureAlgorithm.HS256);

    File secretKeyFile = File.createTempFile("pulsar-test-secret-key-", ".key");
    secretKeyFile.deleteOnExit();
    Files.write(Paths.get(secretKeyFile.toString()), secretKey.getEncoded());

    AuthenticationProviderToken provider = new AuthenticationProviderToken();

    Properties properties = new Properties();
    properties.setProperty(AuthenticationProviderToken.CONF_TOKEN_SECRET_KEY, "file://" + secretKeyFile.toString());

    ServiceConfiguration conf = new ServiceConfiguration();
    conf.setProperties(properties);
    provider.initialize(conf);

    String token = AuthTokenUtils.createToken(secretKey, SUBJECT, Optional.empty());

    // Pulsar protocol auth
    String subject = provider.authenticate(new AuthenticationDataSource() {
        @Override
        public boolean hasDataFromCommand() {
            return true;
        }

        @Override
        public String getCommandData() {
            return token;
        }
    });
    assertEquals(subject, SUBJECT);
    provider.close();
}
 

@Test
public void testInvalidInitialize() throws Exception {
    AuthenticationProviderToken provider = new AuthenticationProviderToken();

    try {
        provider.initialize(new ServiceConfiguration());
        fail("should have failed");
    } catch (IOException e) {
        // Expected, secret key was not defined
    } finally {
        // currently, will not close any resource
        provider.close();
    }
}
 

public Compactor(ServiceConfiguration conf,
                 PulsarClient pulsar,
                 BookKeeper bk,
                 ScheduledExecutorService scheduler) {
    this.conf = conf;
    this.scheduler = scheduler;
    this.pulsar = pulsar;
    this.bk = bk;
}
 

@Test
public void testListenerWithTLSPort() {
    ServiceConfiguration config = new ServiceConfiguration();
    config.setBrokerServicePortTls(Optional.of(6651));
    config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660, internal:pulsar+ssl://127.0.0.1:6651");
    config.setInternalListenerName("internal");
    MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config);
}
 

@Test(expectedExceptions = IllegalArgumentException.class)
public void testDifferentListenerWithSameHostPort() {
    ServiceConfiguration config = new ServiceConfiguration();
    config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660," + " external:pulsar://127.0.0.1:6660");
    config.setInternalListenerName("internal");
    MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config);
}
 

@Test(expectedExceptions = IllegalArgumentException.class)
public void testListenerDuplicate_2() {
    ServiceConfiguration config = new ServiceConfiguration();
    config.setAdvertisedListeners(" internal:pulsar://127.0.0.1:6660," + " internal:pulsar://192.168.1.11:6660");
    config.setInternalListenerName("internal");
    MultipleListenerValidator.validateAndAnalysisAdvertisedListener(config);
}
 

/**
 * @param pulsar
 *              An instance of {@link PulsarService}
 * @param enableTLS
 *              Enable tls or not
 */
public PulsarChannelInitializer(PulsarService pulsar, boolean enableTLS) throws Exception {
    super();
    this.pulsar = pulsar;
    this.enableTls = enableTLS;
    ServiceConfiguration serviceConfig = pulsar.getConfiguration();
    this.tlsEnabledWithKeyStore = serviceConfig.isTlsEnabledWithKeyStore();
    if (this.enableTls) {
        if (tlsEnabledWithKeyStore) {
            nettySSLContextAutoRefreshBuilder = new NettySSLContextAutoRefreshBuilder(
                    serviceConfig.getTlsProvider(),
                    serviceConfig.getTlsKeyStoreType(),
                    serviceConfig.getTlsKeyStore(),
                    serviceConfig.getTlsKeyStorePassword(),
                    serviceConfig.isTlsAllowInsecureConnection(),
                    serviceConfig.getTlsTrustStoreType(),
                    serviceConfig.getTlsTrustStore(),
                    serviceConfig.getTlsTrustStorePassword(),
                    serviceConfig.isTlsRequireTrustedClientCertOnConnect(),
                    serviceConfig.getTlsCiphers(),
                    serviceConfig.getTlsProtocols(),
                    serviceConfig.getTlsCertRefreshCheckDurationSec());
        } else {
            sslCtxRefresher = new NettyServerSslContextBuilder(serviceConfig.isTlsAllowInsecureConnection(),
                    serviceConfig.getTlsTrustCertsFilePath(), serviceConfig.getTlsCertificateFilePath(),
                    serviceConfig.getTlsKeyFilePath(),
                    serviceConfig.getTlsCiphers(), serviceConfig.getTlsProtocols(),
                    serviceConfig.isTlsRequireTrustedClientCertOnConnect(),
                    serviceConfig.getTlsCertRefreshCheckDurationSec());
        }
    } else {
        this.sslCtxRefresher = null;
    }
    this.brokerConf = pulsar.getConfiguration();

    pulsar.getExecutor().scheduleAtFixedRate(safeRun(this::refreshAuthenticationCredentials),
            pulsar.getConfig().getAuthenticationRefreshCheckSeconds(),
            pulsar.getConfig().getAuthenticationRefreshCheckSeconds(), TimeUnit.SECONDS);
}