org.apache.bookkeeper.util.ZkUtils Java Examples

The following examples show how to use org.apache.bookkeeper.util.ZkUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PartitionCreationTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateMissedPartitions() throws JsonProcessingException, KeeperException, InterruptedException, PulsarAdminException, PulsarClientException {
    conf.setAllowAutoTopicCreation(false);
    final String topic = "testCreateMissedPartitions";
    String path = ZkAdminPaths.partitionedTopicPath(TopicName.get(topic));
    int numPartitions = 3;
    byte[] data = jsonMapper().writeValueAsBytes(new PartitionedTopicMetadata(numPartitions));
    // simulate partitioned topic without partitions
    ZkUtils.createFullPathOptimistic(pulsar.getGlobalZkCache().getZooKeeper(), path, data, ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    Consumer<byte[]> consumer = null;
    try {
        consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub-1").subscribeAsync().get(3, TimeUnit.SECONDS);
    } catch (Exception e) {
        //ok here, consumer will create failed with 'Topic does not exist'
    }
    Assert.assertNull(consumer);
    admin.topics().createMissedPartitions(topic);
    consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub-1").subscribe();
    Assert.assertNotNull(consumer);
    Assert.assertTrue(consumer instanceof MultiTopicsConsumerImpl);
    Assert.assertEquals(((MultiTopicsConsumerImpl)consumer).getConsumers().size(), 3);
}
 
Example #2
Source File: MockedZooKeeperClientFactoryImpl.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<ZooKeeper> create(String serverList, SessionType sessionType, int zkSessionTimeoutMillis) {
    MockZooKeeper mockZooKeeper = MockZooKeeper.newInstance();
    createdInstances.add(mockZooKeeper);
    // not used for mock mode
    List<ACL> dummyAclList = new ArrayList<ACL>(0);

    try {
        ZkUtils.createFullPathOptimistic(mockZooKeeper, "/ledgers/available/192.168.1.1:" + 5000,
                "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList, CreateMode.PERSISTENT);

        mockZooKeeper.create("/ledgers/LAYOUT", "1\nflat:1".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME),
                dummyAclList, CreateMode.PERSISTENT);
        return CompletableFuture.completedFuture(mockZooKeeper);

    } catch (KeeperException | InterruptedException e) {
        CompletableFuture<ZooKeeper> future = new CompletableFuture<>();
        future.completeExceptionally(e);
        return future;
    }
}
 
Example #3
Source File: LedgerAllocatorPool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void initializePool() throws IOException {
    try {
        List<String> allocators;
        try {
            allocators = zkc.get().getChildren(poolPath, false);
        } catch (KeeperException.NoNodeException e) {
            logger.info("Allocator Pool {} doesn't exist. Creating it.", poolPath);
            ZkUtils.createFullPathOptimistic(zkc.get(), poolPath, new byte[0], zkc.getDefaultACL(),
                    CreateMode.PERSISTENT);
            allocators = zkc.get().getChildren(poolPath, false);
        }
        if (null == allocators) {
            allocators = new ArrayList<String>();
        }
        if (allocators.size() < corePoolSize) {
            createAllocators(corePoolSize - allocators.size());
            allocators = zkc.get().getChildren(poolPath, false);
        }
        initializeAllocators(allocators);
    } catch (InterruptedException ie) {
        throw new DLInterruptedException("Interrupted when ensuring " + poolPath + " created : ", ie);
    } catch (KeeperException ke) {
        throw new IOException("Encountered zookeeper exception when initializing pool " + poolPath + " : ", ke);
    }
}
 
Example #4
Source File: ZkIsolatedBookieEnsemblePlacementPolicyTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testNoIsolationGroup() throws Exception {
    String data = "{\"group1\": {\"" + BOOKIE1
            + "\": {\"rack\": \"rack0\", \"hostname\": \"bookie1.example.com\"}, \"" + BOOKIE2
            + "\": {\"rack\": \"rack1\", \"hostname\": \"bookie2.example.com\"}}, \"group2\": {\"" + BOOKIE3
            + "\": {\"rack\": \"rack0\", \"hostname\": \"bookie3.example.com\"}, \"" + BOOKIE4
            + "\": {\"rack\": \"rack2\", \"hostname\": \"bookie4.example.com\"}}}";

    ZkUtils.createFullPathOptimistic(localZkc, ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, data.getBytes(),
            ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    Thread.sleep(100);

    ZkIsolatedBookieEnsemblePlacementPolicy isolationPolicy = new ZkIsolatedBookieEnsemblePlacementPolicy();
    ClientConfiguration bkClientConf = new ClientConfiguration();
    bkClientConf.setProperty(ZooKeeperCache.ZK_CACHE_INSTANCE, new ZooKeeperCache("test", localZkc, 30) {
    });
    isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL,
            NullStatsLogger.INSTANCE);
    isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies);

    isolationPolicy.newEnsemble(4, 4, 4, Collections.emptyMap(), new HashSet<>());
}
 
Example #5
Source File: LedgerAllocatorPool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void initializePool() throws IOException {
    try {
        List<String> allocators;
        try {
            allocators = zkc.get().getChildren(poolPath, false);
        } catch (KeeperException.NoNodeException e) {
            logger.info("Allocator Pool {} doesn't exist. Creating it.", poolPath);
            ZkUtils.createFullPathOptimistic(zkc.get(), poolPath, new byte[0], zkc.getDefaultACL(),
                    CreateMode.PERSISTENT);
            allocators = zkc.get().getChildren(poolPath, false);
        }
        if (null == allocators) {
            allocators = new ArrayList<String>();
        }
        if (allocators.size() < corePoolSize) {
            createAllocators(corePoolSize - allocators.size());
            allocators = zkc.get().getChildren(poolPath, false);
        }
        initializeAllocators(allocators);
    } catch (InterruptedException ie) {
        throw new DLInterruptedException("Interrupted when ensuring " + poolPath + " created : ", ie);
    } catch (KeeperException ke) {
        throw new IOException("Encountered zookeeper exception when initializing pool " + poolPath + " : ", ke);
    }
}
 
Example #6
Source File: BookkeeperSchemaStorage.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@NotNull
private CompletableFuture<LocatorEntry> createSchemaLocator(String id, SchemaStorageFormat.SchemaLocator locator) {
    CompletableFuture<LocatorEntry> future = new CompletableFuture<>();

    ZkUtils.asyncCreateFullPathOptimistic(zooKeeper, id, locator.toByteArray(), Acl,
            CreateMode.PERSISTENT, (rc, path, ctx, name) -> {
                Code code = Code.get(rc);
                if (code != Code.OK) {
                    future.completeExceptionally(KeeperException.create(code));
                } else {
                    // Newly created z-node will have version 0
                    future.complete(new LocatorEntry(locator, 0));
                }
            }, null);

    return future;
}
 
Example #7
Source File: DiscoveryServiceTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private void addBrokerToZk(int number) throws Exception {

        for (int i = 0; i < number; i++) {
            LoadReport report = new LoadReport(null, null, "pulsar://broker-:15000" + i, null);
            String reportData = ObjectMapperFactory.getThreadLocal().writeValueAsString(report);
            ZkUtils.createFullPathOptimistic(mockZooKeeper, LOADBALANCE_BROKERS_ROOT + "/" + "broker-" + i,
                    reportData.getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), ZooDefs.Ids.OPEN_ACL_UNSAFE,
                    CreateMode.PERSISTENT);
        }

        // sometimes test-environment takes longer time to trigger async mockZK-watch: so reload cache explicitly
        Field field = ZookeeperCacheLoader.class.getDeclaredField("availableBrokersCache");
        field.setAccessible(true);
        ZooKeeperChildrenCache availableBrokersCache = (ZooKeeperChildrenCache) field
                .get(service.getDiscoveryProvider().localZkCache);
        availableBrokersCache.reloadCache(LOADBALANCE_BROKERS_ROOT);
    }
 
Example #8
Source File: SimpleLoadManagerImplTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
private void createNamespacePolicies(PulsarService pulsar) throws Exception {
    NamespaceIsolationPolicies policies = new NamespaceIsolationPolicies();
    // set up policy that use this broker as primary
    NamespaceIsolationData policyData = new NamespaceIsolationData();
    policyData.namespaces = new ArrayList<String>();
    policyData.namespaces.add("pulsar/use/primary-ns.*");
    policyData.primary = new ArrayList<String>();
    policyData.primary.add(pulsar1.getAdvertisedAddress() + "*");
    policyData.secondary = new ArrayList<String>();
    policyData.secondary.add("prod2-broker([78]).messaging.usw.example.co.*");
    policyData.auto_failover_policy = new AutoFailoverPolicyData();
    policyData.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
    policyData.auto_failover_policy.parameters = new HashMap<String, String>();
    policyData.auto_failover_policy.parameters.put("min_limit", "1");
    policyData.auto_failover_policy.parameters.put("usage_threshold", "100");
    policies.setPolicy("primaryBrokerPolicy", policyData);

    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    ZooKeeper globalZk = pulsar.getGlobalZkCache().getZooKeeper();
    ZkUtils.createFullPathOptimistic(globalZk, AdminResource.path("clusters", "use", "namespaceIsolationPolicies"),
            new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    byte[] content = jsonMapper.writeValueAsBytes(policies.getPolicies());
    globalZk.setData(AdminResource.path("clusters", "use", "namespaceIsolationPolicies"), content, -1);

}
 
Example #9
Source File: NamespacesTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
@Test
public void testValidateAdminAccessOnTenant() throws Exception {
    try {
        final String property = "prop";
        pulsar.getConfiguration().setAuthenticationEnabled(true);
        pulsar.getConfiguration().setAuthorizationEnabled(true);
        final String path = PulsarWebResource.path(POLICIES, property);
        final String data = ObjectMapperFactory.getThreadLocal().writeValueAsString(
                new TenantInfo(Sets.newHashSet(namespaces.clientAppId()), Sets.newHashSet("use")));
        ZkUtils.createFullPathOptimistic(pulsar.getConfigurationCache().getZooKeeper(), path, data.getBytes(),
                ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

        namespaces.validateTenantOperation(property, null);
    } finally {
        pulsar.getConfiguration().setAuthenticationEnabled(false);
        pulsar.getConfiguration().setAuthorizationEnabled(false);
    }
}
 
Example #10
Source File: ModularLoadManagerImplTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
/**
 * It verifies that pulsar-service fails if load-manager tries to create ephemeral znode for broker which is already
 * created by other zk-session-id.
 *
 * @throws Exception
 */
@Test
public void testOwnBrokerZnodeByMultipleBroker() throws Exception {

    ServiceConfiguration config = new ServiceConfiguration();
    config.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
    config.setClusterName("use");
    config.setWebServicePort(Optional.of(0));
    config.setZookeeperServers("127.0.0.1" + ":" + bkEnsemble.getZookeeperPort());
    config.setBrokerServicePort(Optional.of(0));
    PulsarService pulsar = new PulsarService(config);
    // create znode using different zk-session
    final String brokerZnode = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar.getAdvertisedAddress() + ":"
            + config.getWebServicePort();
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), brokerZnode, "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.EPHEMERAL);
    try {
        pulsar.start();
    } catch (PulsarServerException e) {
        //Ok.
    }

    pulsar.close();
}
 
Example #11
Source File: ModularLoadManagerImplTest.java    From pulsar with Apache License 2.0 6 votes vote down vote up
/**
 * It verifies that once broker owns max-number of topics: load-manager doesn't allocates new bundles to that broker
 * unless all the brokers are in same state.
 *
 * <pre>
 * 1. Create a bundle whose bundle-resource-quota will contain max-topics
 * 2. Load-manager assigns broker to this bundle so, assigned broker is overloaded with max-topics
 * 3. For any new further bundles: broker assigns different brokers.
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testMaxTopicDistributionToBroker() throws Exception {

    final int totalBundles = 50;
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(nsFactory, "test", "test", "test",
            totalBundles);
    final BundleData bundleData = new BundleData(10, 1000);
    // it sets max topics under this bundle so, owner of this broker reaches max-topic threshold
    bundleData.setTopics(pulsar1.getConfiguration().getLoadBalancerBrokerMaxTopics() + 10);
    final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000);
    longTermMessageData.setMsgRateIn(1000);
    bundleData.setLongTermData(longTermMessageData);
    final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_ZPATH, bundles[0]);
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), firstBundleDataPath, bundleData.getJsonBytes(),
            ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    String maxTopicOwnedBroker = primaryLoadManager.selectBrokerForAssignment(bundles[0]).get();

    for (int i = 1; i < totalBundles; i++) {
        assertNotEquals(primaryLoadManager.selectBrokerForAssignment(bundles[i]), maxTopicOwnedBroker);
    }
}
 
Example #12
Source File: LoadBalancerTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private void createNamespace(PulsarService pulsar, String namespace, int numBundles) throws Exception {
    Policies policies = new Policies();
    policies.bundles = getBundles(numBundles);

    ObjectMapper jsonMapper = ObjectMapperFactory.create();
    ZooKeeper globalZk = pulsar.getGlobalZkCache().getZooKeeper();
    String zpath = AdminResource.path(POLICIES, namespace);
    ZkUtils.createFullPathOptimistic(globalZk, zpath, jsonMapper.writeValueAsBytes(policies),
            ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

}
 
Example #13
Source File: PersistentTransactionBufferTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public static MockZooKeeper createMockZooKeeper() throws Exception {
    MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.newDirectExecutorService());
    List<ACL> dummyAclList = new ArrayList<>(0);

    ZkUtils.createFullPathOptimistic(zk, "/ledgers/available/192.168.1.1:" + 5000,
                                     "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList, CreateMode.PERSISTENT);

    zk.create("/ledgers/LAYOUT", "1\nflat:1".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList,
              CreateMode.PERSISTENT);
    return zk;
}
 
Example #14
Source File: KopProtocolHandlerTestBase.java    From kop with Apache License 2.0 5 votes vote down vote up
public static MockZooKeeper createMockZooKeeper() throws Exception {
    MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.newDirectExecutorService());
    List<ACL> dummyAclList = new ArrayList<>(0);

    ZkUtils.createFullPathOptimistic(zk, "/ledgers/available/192.168.1.1:" + 5000,
        "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList, CreateMode.PERSISTENT);

    zk.create(
        "/ledgers/LAYOUT",
        "1\nflat:1".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList,
        CreateMode.PERSISTENT);
    return zk;
}
 
Example #15
Source File: ZkIsolatedBookieEnsemblePlacementPolicyTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoBookieInfo() throws Exception {
    ZkIsolatedBookieEnsemblePlacementPolicy isolationPolicy = new ZkIsolatedBookieEnsemblePlacementPolicy();
    ClientConfiguration bkClientConf = new ClientConfiguration();
    bkClientConf.setProperty(ZooKeeperCache.ZK_CACHE_INSTANCE, new ZooKeeperCache("test", localZkc, 30) {
    });
    bkClientConf.setProperty(ZkIsolatedBookieEnsemblePlacementPolicy.ISOLATION_BOOKIE_GROUPS, isolationGroups);
    isolationPolicy.initialize(bkClientConf, Optional.empty(), timer, SettableFeatureProvider.DISABLE_ALL, NullStatsLogger.INSTANCE);
    isolationPolicy.onClusterChanged(writableBookies, readOnlyBookies);

    isolationPolicy.newEnsemble(4, 4, 4, Collections.emptyMap(), new HashSet<>());

    String data = "{\"group1\": {\"" + BOOKIE1
            + "\": {\"rack\": \"rack0\", \"hostname\": \"bookie1.example.com\"}, \"" + BOOKIE2
            + "\": {\"rack\": \"rack1\", \"hostname\": \"bookie2.example.com\"}}, \"group2\": {\"" + BOOKIE3
            + "\": {\"rack\": \"rack0\", \"hostname\": \"bookie3.example.com\"}, \"" + BOOKIE4
            + "\": {\"rack\": \"rack2\", \"hostname\": \"bookie4.example.com\"}}}";

    ZkUtils.createFullPathOptimistic(localZkc, ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, data.getBytes(),
            ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    Thread.sleep(100);

    List<BookieSocketAddress> ensemble = isolationPolicy.newEnsemble(2, 2, 2, Collections.emptyMap(), new HashSet<>()).getResult();
    assertTrue(ensemble.contains(new BookieSocketAddress(BOOKIE1)));
    assertTrue(ensemble.contains(new BookieSocketAddress(BOOKIE2)));

    try {
        isolationPolicy.newEnsemble(3, 3, 3, Collections.emptyMap(), new HashSet<>());
        fail("should not pass");
    } catch (BKNotEnoughBookiesException e) {
        // ok
    }

    localZkc.delete(ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, -1);
}
 
Example #16
Source File: ModularLoadManagerImplTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test(enabled = false)
public void testEvenBundleDistribution() throws Exception {
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils.makeBundles(nsFactory, "test", "test", "test", 16);
    int numAssignedToPrimary = 0;
    int numAssignedToSecondary = 0;
    final BundleData bundleData = new BundleData(10, 1000);
    final TimeAverageMessageData longTermMessageData = new TimeAverageMessageData(1000);
    longTermMessageData.setMsgRateIn(1000);
    bundleData.setLongTermData(longTermMessageData);
    final String firstBundleDataPath = String.format("%s/%s", ModularLoadManagerImpl.BUNDLE_DATA_ZPATH, bundles[0]);
    // Write long message rate for first bundle to ensure that even bundle distribution is not a coincidence of
    // balancing by message rate. If we were balancing by message rate, one of the brokers should only have this
    // one bundle.
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), firstBundleDataPath, bundleData.getJsonBytes(),
            ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    for (final NamespaceBundle bundle : bundles) {
        if (primaryLoadManager.selectBrokerForAssignment(bundle).equals(primaryHost)) {
            ++numAssignedToPrimary;
        } else {
            ++numAssignedToSecondary;
        }
        if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) {
            // On even number of assignments, assert that an equal number of bundles have been assigned between
            // them.
            assertEquals(numAssignedToPrimary, numAssignedToSecondary);
        }
    }
}
 
Example #17
Source File: ModularLoadManagerImplTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testZnodeMissed() throws Exception {
    String path = LoadManager.LOADBALANCE_BROKERS_ROOT + "/" + pulsar1.getAdvertisedAddress() + ":" + pulsar1.getConfiguration().getWebServicePort().get();
    ZkUtils.deleteFullPathOptimistic(pulsar1.getZkClient(), path, -1);
    pulsar1.getLoadManager().get().writeLoadReportOnZookeeper();
    // Delete it again to check the znode is create before write load balance data
    ZkUtils.deleteFullPathOptimistic(pulsar1.getZkClient(), path, -1);
}
 
Example #18
Source File: ServiceUnitZkUtils.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private static void createZnodeOptimistic(ZooKeeper zkc, String path, String data, CreateMode mode)
        throws KeeperException, InterruptedException {
    try {
        // create node optimistically
        checkNotNull(LocalZooKeeperConnectionService.createIfAbsent(zkc, path, data, mode));
    } catch (NoNodeException e) {
        // if path contains multiple levels after the root, create the intermediate nodes first
        String[] parts = path.split("/");
        if (parts.length > 3) {
            String int_path = path.substring(0, path.lastIndexOf("/"));
            if (zkc.exists(int_path, false) == null) {
                // create the intermediate nodes
                try {
                    ZkUtils.createFullPathOptimistic(zkc, int_path, new byte[0], Ids.OPEN_ACL_UNSAFE,
                            CreateMode.PERSISTENT);
                } catch (KeeperException.NodeExistsException nee) {
                    LOG.debug(
                            "Other broker preempted the full intermediate path [{}] already. Continue for acquiring the leaf ephemeral node.",
                            int_path);
                }
            }
            checkNotNull(LocalZooKeeperConnectionService.createIfAbsent(zkc, path, data, mode));
        } else {
            // If failed to create immediate child of root node, throw exception
            throw e;
        }
    }
}
 
Example #19
Source File: ZkBookieRackAffinityMappingTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testNoBookieInfo() throws Exception {
    ZkBookieRackAffinityMapping mapping = new ZkBookieRackAffinityMapping();
    ClientConfiguration bkClientConf = new ClientConfiguration();
    bkClientConf.setProperty(ZooKeeperCache.ZK_CACHE_INSTANCE, new ZooKeeperCache("test", localZkc, 30) {
    });
    mapping.setConf(bkClientConf);
    List<String> racks = mapping.resolve(Lists.newArrayList("127.0.0.1", "127.0.0.2", "127.0.0.3"));
    assertEquals(racks.get(0), null);
    assertEquals(racks.get(1), null);
    assertEquals(racks.get(2), null);

    Map<String, Map<BookieSocketAddress, BookieInfo>> bookieMapping = new HashMap<>();
    Map<BookieSocketAddress, BookieInfo> mainBookieGroup = new HashMap<>();

    mainBookieGroup.put(BOOKIE1, new BookieInfo("/rack0", null));
    mainBookieGroup.put(BOOKIE2, new BookieInfo("/rack1", null));

    bookieMapping.put("group1", mainBookieGroup);

    ZkUtils.createFullPathOptimistic(localZkc, ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH,
            jsonMapper.writeValueAsBytes(bookieMapping), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    Thread.sleep(100);

    racks = mapping.resolve(Lists.newArrayList("127.0.0.1", "127.0.0.2", "127.0.0.3"));
    assertEquals(racks.get(0), "/rack0");
    assertEquals(racks.get(1), "/rack1");
    assertEquals(racks.get(2), null);

    localZkc.delete(ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, -1);
}
 
Example #20
Source File: SimpleLoadManagerImplTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testEvenBundleDistribution() throws Exception {
    final NamespaceBundle[] bundles = LoadBalancerTestingUtils
            .makeBundles(pulsar1.getNamespaceService().getNamespaceBundleFactory(), "pulsar", "use", "test", 16);
    final ResourceQuota quota = new ResourceQuota();
    final String quotaZPath = String.format("%s/%s/%s", ResourceQuotaCache.RESOURCE_QUOTA_ROOT, "namespace",
            bundles[0]);
    // Create high message rate quota for the first bundle to make it unlikely to be a coincidence of even
    // distribution.
    ZkUtils.createFullPathOptimistic(pulsar1.getZkClient(), quotaZPath,
            ObjectMapperFactory.getThreadLocal().writeValueAsBytes(quota), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.PERSISTENT);
    int numAssignedToPrimary = 0;
    int numAssignedToSecondary = 0;
    pulsar1.getConfiguration().setLoadBalancerPlacementStrategy(SimpleLoadManagerImpl.LOADBALANCER_STRATEGY_LLS);
    final SimpleLoadManagerImpl loadManager = (SimpleLoadManagerImpl) pulsar1.getLoadManager().get();

    for (final NamespaceBundle bundle : bundles) {
        if (loadManager.getLeastLoaded(bundle).get().getResourceId().equals(primaryHost)) {
            ++numAssignedToPrimary;
        } else {
            ++numAssignedToSecondary;
        }
        // Check that number of assigned bundles are equivalent when an even number have been assigned.
        if ((numAssignedToPrimary + numAssignedToSecondary) % 2 == 0) {
            assert (numAssignedToPrimary == numAssignedToSecondary);
        }
    }
}
 
Example #21
Source File: MockedPulsarServiceBaseTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public static MockZooKeeper createMockZooKeeper() throws Exception {
    MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.newDirectExecutorService());
    List<ACL> dummyAclList = new ArrayList<>(0);

    ZkUtils.createFullPathOptimistic(zk, "/ledgers/available/192.168.1.1:" + 5000,
            "".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList, CreateMode.PERSISTENT);

    zk.create("/ledgers/LAYOUT", "1\nflat:1".getBytes(ZookeeperClientFactoryImpl.ENCODING_SCHEME), dummyAclList,
            CreateMode.PERSISTENT);
    return zk;
}
 
Example #22
Source File: BrokerServiceThrottlingTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private void upsertLookupPermits(int permits) throws Exception {
    Map<String, String> throttlingMap = Maps.newHashMap();
    throttlingMap.put("maxConcurrentLookupRequest", Integer.toString(permits));
    byte[] content = ObjectMapperFactory.getThreadLocal().writeValueAsBytes(throttlingMap);
    if (mockZooKeeper.exists(BROKER_SERVICE_CONFIGURATION_PATH, false) != null) {
        mockZooKeeper.setData(BROKER_SERVICE_CONFIGURATION_PATH, content, -1);
    } else {
        ZkUtils.createFullPathOptimistic(mockZooKeeper, BROKER_SERVICE_CONFIGURATION_PATH, content,
                ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
    }
}
 
Example #23
Source File: NamespaceServiceTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
/**
 * <pre>
 *  It verifies that namespace service deserialize the load-report based on load-manager which active.
 *  1. write candidate1- load report using {@link LoadReport} which is used by SimpleLoadManagerImpl
 *  2. Write candidate2- load report using {@link LocalBrokerData} which is used by ModularLoadManagerImpl
 *  3. try to get Lookup Result based on active load-manager
 * </pre>
 * @throws Exception
 */
@Test
public void testLoadReportDeserialize() throws Exception {

    final String candidateBroker1 = "http://localhost:8000";
    final String candidateBroker2 = "http://localhost:3000";
    LoadReport lr = new LoadReport(null, null, candidateBroker1, null);
    LocalBrokerData ld = new LocalBrokerData(null, null, candidateBroker2, null);
    URI uri1 = new URI(candidateBroker1);
    URI uri2 = new URI(candidateBroker2);
    String path1 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri1.getHost(), uri1.getPort());
    String path2 = String.format("%s/%s:%s", LoadManager.LOADBALANCE_BROKERS_ROOT, uri2.getHost(), uri2.getPort());
    ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), path1,
            ObjectMapperFactory.getThreadLocal().writeValueAsBytes(lr), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.EPHEMERAL);
    ZkUtils.createFullPathOptimistic(pulsar.getZkClient(), path2,
            ObjectMapperFactory.getThreadLocal().writeValueAsBytes(ld), ZooDefs.Ids.OPEN_ACL_UNSAFE,
            CreateMode.EPHEMERAL);
    LookupResult result1 = pulsar.getNamespaceService().createLookupResult(candidateBroker1, false).get();

    // update to new load manager
    LoadManager oldLoadManager = pulsar.getLoadManager()
            .getAndSet(new ModularLoadManagerWrapper(new ModularLoadManagerImpl()));
    oldLoadManager.stop();
    LookupResult result2 = pulsar.getNamespaceService().createLookupResult(candidateBroker2, false).get();
    Assert.assertEquals(result1.getLookupData().getBrokerUrl(), candidateBroker1);
    Assert.assertEquals(result2.getLookupData().getBrokerUrl(), candidateBroker2);
    System.out.println(result2);
}
 
Example #24
Source File: ZkBookieRackAffinityMappingTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@Test
public void testBasic() throws Exception {
    String data = "{\"group1\": {\"" + BOOKIE1
            + "\": {\"rack\": \"/rack0\", \"hostname\": \"bookie1.example.com\"}, \"" + BOOKIE2
            + "\": {\"rack\": \"/rack1\", \"hostname\": \"bookie2.example.com\"}}}";
    ZkUtils.createFullPathOptimistic(localZkc, ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, data.getBytes(),
            ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);

    // Case1: ZKCache is given
    ZkBookieRackAffinityMapping mapping1 = new ZkBookieRackAffinityMapping();
    ClientConfiguration bkClientConf1 = new ClientConfiguration();
    bkClientConf1.setProperty(ZooKeeperCache.ZK_CACHE_INSTANCE, new ZooKeeperCache("test", localZkc, 30) {
    });
    mapping1.setConf(bkClientConf1);
    List<String> racks1 = mapping1
            .resolve(Lists.newArrayList(BOOKIE1.getHostName(), BOOKIE2.getHostName(), BOOKIE3.getHostName()));
    assertEquals(racks1.get(0), "/rack0");
    assertEquals(racks1.get(1), "/rack1");
    assertEquals(racks1.get(2), null);

    // Case 2: ZkServers and ZkTimeout are given (ZKCache will be constructed in
    // ZkBookieRackAffinityMapping#setConf)
    ZkBookieRackAffinityMapping mapping2 = new ZkBookieRackAffinityMapping();
    ClientConfiguration bkClientConf2 = new ClientConfiguration();
    bkClientConf2.setZkServers("127.0.0.1" + ":" + localZkS.getZookeeperPort());
    bkClientConf2.setZkTimeout(1000);
    mapping2.setConf(bkClientConf2);
    List<String> racks2 = mapping2
            .resolve(Lists.newArrayList(BOOKIE1.getHostName(), BOOKIE2.getHostName(), BOOKIE3.getHostName()));
    assertEquals(racks2.get(0), "/rack0");
    assertEquals(racks2.get(1), "/rack1");
    assertEquals(racks2.get(2), null);

    localZkc.delete(ZkBookieRackAffinityMapping.BOOKIE_INFO_ROOT_PATH, -1);
}
 
Example #25
Source File: OwnershipCache.java    From pulsar with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
@Override
public CompletableFuture<OwnedBundle> asyncLoad(String namespaceBundleZNode, Executor executor) {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Acquiring zk lock on namespace {}", namespaceBundleZNode);
    }

    byte[] znodeContent;
    try {
        znodeContent = jsonMapper.writeValueAsBytes(selfOwnerInfo);
    } catch (JsonProcessingException e) {
        // Failed to serialize to JSON
        return FutureUtil.failedFuture(e);
    }

    CompletableFuture<OwnedBundle> future = new CompletableFuture<>();
    ZkUtils.asyncCreateFullPathOptimistic(localZkCache.getZooKeeper(), namespaceBundleZNode, znodeContent,
            Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, (rc, path, ctx, name) -> {
                if (rc == KeeperException.Code.OK.intValue()) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Successfully acquired zk lock on {}", namespaceBundleZNode);
                    }
                    ownershipReadOnlyCache.invalidate(namespaceBundleZNode);
                    future.complete(new OwnedBundle(
                            ServiceUnitZkUtils.suBundleFromPath(namespaceBundleZNode, bundleFactory)));
                } else {
                    // Failed to acquire lock
                    future.completeExceptionally(KeeperException.create(rc));
                }
            }, null);

    return future;
}
 
Example #26
Source File: MockedBookKeeperTestCase.java    From pulsar with Apache License 2.0 5 votes vote down vote up
/**
 * Start cluster
 *
 * @throws Exception
 */
protected void startBookKeeper() throws Exception {
    zkc = MockZooKeeper.newInstance();
    for (int i = 0; i < numBookies; i++) {
        ZkUtils.createFullPathOptimistic(zkc, "/ledgers/available/192.168.1.1:" + (5000 + i), "".getBytes(), null,
                null);
    }

    zkc.create("/ledgers/LAYOUT", "1\nflat:1".getBytes(), null, null);

    bkc = new PulsarMockBookKeeper(zkc, executor.chooseThread(this));
}
 
Example #27
Source File: TestZKLogStreamMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
    zkc = TestZooKeeperClientBuilder.newBuilder()
            .name("zkc")
            .uri(DLMTestUtil.createDLMURI(zkPort, "/"))
            .sessionTimeoutMs(sessionTimeoutMs)
            .build();
    uri = DLMTestUtil.createDLMURI(zkPort, "");
    try {
        ZkUtils.createFullPathOptimistic(
                zkc.get(),
                uri.getPath(),
                new byte[0],
                ZooDefs.Ids.OPEN_ACL_UNSAFE,
                CreateMode.PERSISTENT);
    } catch (KeeperException.NodeExistsException nee) {
        logger.debug("The namespace uri already exists.");
    }
    scheduler = OrderedScheduler.newBuilder()
        .name("test-scheduler")
        .corePoolSize(1)
        .build();
    metadataStore = new ZKLogStreamMetadataStore(
        "test-logstream-metadata-store",
        new DistributedLogConfiguration(),
        zkc,
        scheduler,
        NullStatsLogger.INSTANCE);
}
 
Example #28
Source File: TestZKLogStreamMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testGetMissingPathsRecursive2() throws Exception {
    String path = uri.getPath() + "/path/to/log";
    ZkUtils.createFullPathOptimistic(
        zkc.get(), path, EMPTY_BYTES, zkc.getDefaultACL(), CreateMode.PERSISTENT);

    List<String> missingPaths = FutureUtils.result(
        getMissingPaths(zkc, uri, "path/to/log"));

    assertEquals(
        Collections.emptyList(),
        missingPaths);
}
 
Example #29
Source File: BlobStoreManagedLedgerOffloaderTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private static MockZooKeeper createMockZooKeeper() throws Exception {
    MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.newDirectExecutorService());
    List<ACL> dummyAclList = new ArrayList<ACL>(0);

    ZkUtils.createFullPathOptimistic(zk, "/ledgers/available/192.168.1.1:" + 5000,
            "".getBytes(UTF_8), dummyAclList, CreateMode.PERSISTENT);

    zk.create("/ledgers/LAYOUT", "1\nflat:1".getBytes(UTF_8), dummyAclList,
            CreateMode.PERSISTENT);
    return zk;
}
 
Example #30
Source File: FileSystemManagedLedgerOffloaderTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
private static MockZooKeeper createMockZooKeeper() throws Exception {
    MockZooKeeper zk = MockZooKeeper.newInstance(MoreExecutors.newDirectExecutorService());
    List<ACL> dummyAclList = new ArrayList<ACL>(0);

    ZkUtils.createFullPathOptimistic(zk, "/ledgers/available/192.168.1.1:" + 5000,
            "".getBytes(UTF_8), dummyAclList, CreateMode.PERSISTENT);

    zk.create("/ledgers/LAYOUT", "1\nflat:1".getBytes(UTF_8), dummyAclList,
            CreateMode.PERSISTENT);
    return zk;
}