/** * Copyright 2016 Netflix, Inc. * <p> * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.netflix.dyno.connectionpool.impl.lb; import com.netflix.dyno.connectionpool.BaseOperation; import com.netflix.dyno.connectionpool.Connection; import com.netflix.dyno.connectionpool.ConnectionPoolConfiguration.LoadBalancingStrategy; import com.netflix.dyno.connectionpool.ConnectionPoolMonitor; import com.netflix.dyno.connectionpool.HashPartitioner; import com.netflix.dyno.connectionpool.Host; import com.netflix.dyno.connectionpool.Host.Status; import com.netflix.dyno.connectionpool.HostBuilder; import com.netflix.dyno.connectionpool.HostConnectionPool; import com.netflix.dyno.connectionpool.RetryPolicy; import com.netflix.dyno.connectionpool.TokenMapSupplier; import com.netflix.dyno.connectionpool.impl.ConnectionPoolConfigurationImpl; import com.netflix.dyno.connectionpool.impl.CountingConnectionPoolMonitor; import com.netflix.dyno.connectionpool.impl.RetryNTimes; import com.netflix.dyno.connectionpool.impl.utils.CollectionUtils; import com.netflix.dyno.connectionpool.impl.utils.CollectionUtils.Transform; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; import static org.mockito.Matchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class HostSelectionWithFallbackTest { private Map<Host, AtomicBoolean> poolStatus = new HashMap<Host, AtomicBoolean>(); private BaseOperation<Integer, Integer> testOperation = new BaseOperation<Integer, Integer>() { @Override public String getName() { return "test"; } @Override public String getStringKey() { return "11"; } @Override public byte[] getBinaryKey() { return null; } }; private final ConnectionPoolConfigurationImpl cpConfig = new ConnectionPoolConfigurationImpl("test"); private final ConnectionPoolMonitor cpMonitor = new CountingConnectionPoolMonitor(); String dc = "us-east-1"; String rack1 = "us-east-1c"; String rack2 = "us-east-1d"; String rack3 = "us-east-1e"; Host h1 = new HostBuilder().setHostname("h1").setRack(rack1).setStatus(Status.Up).createHost(); Host h2 = new HostBuilder().setHostname("h2").setRack(rack1).setStatus(Status.Up).createHost(); Host h3 = new HostBuilder().setHostname("h3").setRack("remoteRack1").setStatus(Status.Up).createHost(); Host h4 = new HostBuilder().setHostname("h4").setRack("remoteRack1").setStatus(Status.Up).createHost(); Host h5 = new HostBuilder().setHostname("h5").setRack("remoteRack2").setStatus(Status.Up).createHost(); Host h6 = new HostBuilder().setHostname("h6").setRack("remoteRack2").setStatus(Status.Up).createHost(); Host[] arr = {h1, h2, h3, h4, h5, h6}; List<Host> hosts = Arrays.asList(arr); @Before public void beforeTest() { cpConfig.setLocalRack(rack1); cpConfig.setLocalDataCenter(dc); cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.RoundRobin); cpConfig.withTokenSupplier(getTokenMapSupplier()); } @Test public void testFallbackToRemotePoolWhenPoolInactive() throws Exception { HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); Set<String> hostnames = new HashSet<String>(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h1", "h2"); // Now mark h1 and h2 both as "DOWN" poolStatus.get(h1).set(false); poolStatus.get(h2).set(false); hostnames.clear(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h3", "h4", "h5", "h6"); // Now bring h1 back up poolStatus.get(h1).set(true); hostnames.clear(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h1"); // Now bring h2 back up poolStatus.get(h2).set(true); hostnames.clear(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h1", "h2"); } @Test public void testFallbackToRemotePoolWhenHostDown() throws Exception { HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); Set<String> hostnames = new HashSet<String>(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h1", "h2"); // Now mark h1 and h2 both as "DOWN" h1.setStatus(Status.Down); h2.setStatus(Status.Down); hostnames.clear(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h3", "h4", "h5", "h6"); // Now bring h1 back up h1.setStatus(Status.Up); hostnames.clear(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } System.out.println(" " + hostnames); verifyExactly(hostnames, "h1"); // Now bring h2 back up h2.setStatus(Status.Up); hostnames.clear(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnection(testOperation, 1, TimeUnit.MILLISECONDS); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h1", "h2"); // Verify that failover metric has increased Assert.assertTrue(cpMonitor.getFailoverCount() > 0); } @Test public void testCrossRackFallback() throws Exception { HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); RetryPolicy retry = new RetryNTimes(3, true); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); Set<String> hostnames = new HashSet<String>(); for (int i = 0; i < 10; i++) { Connection<Integer> conn = selection.getConnectionUsingRetryPolicy(testOperation, 1, TimeUnit.MILLISECONDS, retry); hostnames.add(conn.getHost().getHostAddress()); } verifyExactly(hostnames, "h1", "h2"); // Record a failure so that retry attempt is not 0 and get another connection retry.failure(new Exception("Unit Test Retry Exception")); Connection<Integer> conn = selection.getConnectionUsingRetryPolicy(testOperation, 1, TimeUnit.MILLISECONDS, retry); String fallbackHost = conn.getHost().getHostAddress(); Assert.assertTrue(!fallbackHost.equals("h1") && !fallbackHost.equals("h2")); } @Test public void testGetConnectionsFromRingNormal() throws Exception { HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); Collection<String> hostnames = runConnectionsToRingTest(selection); verifyExactly(hostnames, "h1", "h2"); } @Test public void testGetConnectionsFromRingWhenPrimaryHostPoolInactive() throws Exception { HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); // Put Host H1 as DOWN poolStatus.get(h1).set(false); Collection<String> hostnames = runConnectionsToRingTest(selection); verifyPresent(hostnames, "h2"); verifyAtLeastOnePresent(hostnames, "h3", "h5"); // Put Host H2 as DOWN selection.initWithHosts(pools); poolStatus.get(h1).set(true); poolStatus.get(h2).set(false); hostnames = runConnectionsToRingTest(selection); verifyPresent(hostnames, "h1"); verifyAtLeastOnePresent(hostnames, "h4", "h6"); // Put Hosts H1 and H2 as DOWN selection.initWithHosts(pools); poolStatus.get(h1).set(false); poolStatus.get(h2).set(false); hostnames = runConnectionsToRingTest(selection); verifyAtLeastOnePresent(hostnames, "h3", "h5"); verifyAtLeastOnePresent(hostnames, "h4", "h6"); // Put Hosts H1,H2,H3 as DOWN selection.initWithHosts(pools); poolStatus.get(h1).set(false); poolStatus.get(h2).set(false); poolStatus.get(h3).set(false); hostnames = runConnectionsToRingTest(selection); verifyAtLeastOnePresent(hostnames, "h4", "h6"); verifyPresent(hostnames, "h5"); // Put Hosts H1,H2,H3,H4 as DOWN selection.initWithHosts(pools); poolStatus.get(h1).set(false); poolStatus.get(h2).set(false); poolStatus.get(h3).set(false); poolStatus.get(h4).set(false); hostnames = runConnectionsToRingTest(selection); verifyExactly(hostnames, "h5", "h6"); } @Test public void testGetConnectionsFromRingWhenHostDown() throws Exception { HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); // Put Host H1 as DOWN h1.setStatus(Status.Down); Collection<String> hostnames = runConnectionsToRingTest(selection); verifyPresent(hostnames, "h2"); verifyAtLeastOnePresent(hostnames, "h3", "h5"); // Put Host H2 as DOWN selection.initWithHosts(pools); h1.setStatus(Status.Up); h2.setStatus(Status.Down); hostnames = runConnectionsToRingTest(selection); verifyPresent(hostnames, "h1"); verifyAtLeastOnePresent(hostnames, "h4", "h6"); // Put Hosts H1 and H2 as DOWN selection.initWithHosts(pools); h1.setStatus(Status.Down); h2.setStatus(Status.Down); hostnames = runConnectionsToRingTest(selection); verifyAtLeastOnePresent(hostnames, "h3", "h5"); verifyAtLeastOnePresent(hostnames, "h4", "h6"); // Put Hosts H1,H2,H3 as DOWN selection.initWithHosts(pools); h1.setStatus(Status.Down); h2.setStatus(Status.Down); h3.setStatus(Status.Down); hostnames = runConnectionsToRingTest(selection); verifyAtLeastOnePresent(hostnames, "h4", "h6"); verifyPresent(hostnames, "h5"); // Put Hosts H1,H2,H3,H4 as DOWN selection.initWithHosts(pools); h1.setStatus(Status.Down); h2.setStatus(Status.Down); h3.setStatus(Status.Down); h4.setStatus(Status.Down); hostnames = runConnectionsToRingTest(selection); verifyExactly(hostnames, "h5", "h6"); } @Test public void testReplicationFactorOf3WithDupes() { cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); List<HostToken> hostTokens = Arrays.asList( new HostToken(1383429731L, new HostBuilder().setHostname("host-1").setPort(-1).setRack(rack1).createHost()), // Use -1 otherwise the port is opened which works new HostToken(2815085496L, new HostBuilder().setHostname("host-2").setPort(-1).setRack(rack1).createHost()), new HostToken(4246741261L, new HostBuilder().setHostname("host-3").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-4").setPort(-1).setRack(rack1).createHost()), new HostToken(2815085496L, new HostBuilder().setHostname("host-5").setPort(-1).setRack(rack1).createHost()), new HostToken(4246741261L, new HostBuilder().setHostname("host-6").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-7").setPort(-1).setRack(rack1).createHost()), new HostToken(2815085496L, new HostBuilder().setHostname("host-8").setPort(-1).setRack(rack1).createHost()), new HostToken(4246741261L, new HostBuilder().setHostname("host-9").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-7").setPort(-1).setRack(rack1).createHost()), new HostToken(2815085496L, new HostBuilder().setHostname("host-8").setPort(-1).setRack(rack1).createHost()), new HostToken(4246741261L, new HostBuilder().setHostname("host-9").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-7").setPort(-1).setRack(rack1).createHost()), new HostToken(2815085496L, new HostBuilder().setHostname("host-8").setPort(-1).setRack(rack1).createHost()), new HostToken(4246741261L, new HostBuilder().setHostname("host-9").setPort(-1).setRack(rack1).createHost()) ); int rf = selection.calculateReplicationFactor(hostTokens); Assert.assertEquals(3, rf); } @Test public void testReplicationFactorOf3() { cpConfig.setLocalRack("us-east-1c"); cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); List<HostToken> hostTokens = Arrays.asList( new HostToken(1111L, new HostBuilder().setHostname("host-1").setPort(-1).setRack(rack1).createHost()), new HostToken(1111L, new HostBuilder().setHostname("host-2").setPort(-1).setRack(rack1).createHost()), new HostToken(1111L, new HostBuilder().setHostname("host-3").setPort(-1).setRack(rack1).createHost()), new HostToken(2222L, new HostBuilder().setHostname("host-4").setPort(-1).setRack(rack1).createHost()), new HostToken(2222L, new HostBuilder().setHostname("host-5").setPort(-1).setRack(rack1).createHost()), new HostToken(2222L, new HostBuilder().setHostname("host-6").setPort(-1).setRack(rack1).createHost()) ); int rf = selection.calculateReplicationFactor(hostTokens); Assert.assertEquals(3, rf); } @Test public void testReplicationFactorOf2() { cpConfig.setLocalRack(rack1); cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); List<HostToken> hostTokens = Arrays.asList( new HostToken(1111L, new HostBuilder().setHostname("host-1").setPort(-1).setRack(rack1).createHost()), new HostToken(1111L, new HostBuilder().setHostname("host-2").setPort(-1).setRack(rack1).createHost()), new HostToken(2222L, new HostBuilder().setHostname("host-4").setPort(-1).setRack(rack1).createHost()), new HostToken(2222L, new HostBuilder().setHostname("host-5").setPort(-1).setRack(rack1).createHost()) ); int rf = selection.calculateReplicationFactor(hostTokens); Assert.assertEquals(2, rf); } @Test public void testReplicationFactorOf1() { cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); List<HostToken> hostTokens = Arrays.asList( new HostToken(1111L, h1), new HostToken(2222L, h2), new HostToken(3333L, h3), new HostToken(4444L, h4) ); int rf = selection.calculateReplicationFactor(hostTokens); Assert.assertEquals(1, rf); } @Test(expected = RuntimeException.class) public void testIllegalReplicationFactor() { cpConfig.setLocalRack("us-east-1c"); cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); List<HostToken> hostTokens = Arrays.asList( new HostToken(1111L, new HostBuilder().setHostname("host-1").setPort(-1).setRack(rack1).createHost()), new HostToken(1111L, new HostBuilder().setHostname("host-2").setPort(-1).setRack(rack1).createHost()), new HostToken(2222L, new HostBuilder().setHostname("host-4").setPort(-1).setRack(rack1).createHost()) ); selection.calculateReplicationFactor(hostTokens); } @Test public void testReplicationFactorForMultiRegionCluster() { cpConfig.setLocalRack("us-east-1d"); cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); List<HostToken> hostTokens = Arrays.asList( new HostToken(3530913378L, new HostBuilder().setHostname("host-1").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-1").setPort(-1).setRack(rack1).createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-2").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-2").setPort(-1).setRack(rack1).createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-3").setPort(-1).setRack(rack1).createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-3").setPort(-1).setRack(rack1).createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-4").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-4").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-5").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-5").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-6").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-6").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-7").setPort(-1).setRack("remoteRack2").createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-7").setPort(-1).setRack("remoteRack2").createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-8").setPort(-1).setRack("remoteRack2").createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-8").setPort(-1).setRack("remoteRack1").createHost()), new HostToken(3530913378L, new HostBuilder().setHostname("host-9").setPort(-1).setRack("remoteRack2").createHost()), new HostToken(1383429731L, new HostBuilder().setHostname("host-9").setPort(-1).setRack("remoteRack2").createHost()) ); int rf = selection.calculateReplicationFactor(hostTokens); Assert.assertEquals(3, rf); cpConfig.setLocalRack(null); selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); rf = selection.calculateReplicationFactor(hostTokens); Assert.assertEquals(3, rf); } @Test public void testChangingHashPartitioner() { cpConfig.setLoadBalancingStrategy(LoadBalancingStrategy.TokenAware); cpConfig.withTokenSupplier(getTokenMapSupplier()); cpConfig.withHashPartitioner(getMockHashPartitioner(1000000000L)); HostSelectionWithFallback<Integer> selection = new HostSelectionWithFallback<Integer>(cpConfig, cpMonitor); Map<Host, HostConnectionPool<Integer>> pools = new HashMap<Host, HostConnectionPool<Integer>>(); for (Host host : hosts) { poolStatus.put(host, new AtomicBoolean(true)); pools.put(host, getMockHostConnectionPool(host, poolStatus.get(host))); } selection.initWithHosts(pools); Connection<Integer> connection = selection.getConnection(testOperation, 10, TimeUnit.MILLISECONDS); // Verify that h1 has been selected instead of h2 assertEquals("h1", connection.getHost().getHostAddress()); } private Collection<String> runConnectionsToRingTest(HostSelectionWithFallback<Integer> selection) { Collection<Connection<Integer>> connections = selection.getConnectionsToRing(null, 10, TimeUnit.MILLISECONDS); return CollectionUtils.transform(connections, new Transform<Connection<Integer>, String>() { @Override public String get(Connection<Integer> x) { return x.getHost().getHostAddress(); } }); } private void verifyExactly(Collection<String> resultCollection, String... hostnames) { Set<String> result = new HashSet<String>(resultCollection); Set<String> all = new HashSet<String>(); all.add("h1"); all.add("h2"); all.add("h3"); all.add("h4"); all.add("h5"); all.add("h6"); Set<String> expected = new HashSet<String>(Arrays.asList(hostnames)); Set<String> notExpected = new HashSet<String>(all); notExpected.removeAll(expected); for (String e : expected) { Assert.assertTrue("Result: " + result + ", expected: " + e, result.contains(e)); } for (String ne : notExpected) { Assert.assertFalse("Result: " + result, result.contains(ne)); } } private void verifyPresent(Collection<String> resultCollection, String... hostnames) { Set<String> result = new HashSet<String>(resultCollection); for (String h : hostnames) { Assert.assertTrue("Result: " + result + ", expected: " + h, result.contains(h)); } } private void verifyAtLeastOnePresent(Collection<String> resultCollection, String... hostnames) { Set<String> result = new HashSet<String>(resultCollection); boolean present = false; for (String h : hostnames) { if (result.contains(h)) { present = true; break; } } Assert.assertTrue("Result: " + result + ", expected at least one of: " + hostnames, present); } @SuppressWarnings("unchecked") private HostConnectionPool<Integer> getMockHostConnectionPool(final Host host, final AtomicBoolean status) { Connection<Integer> mockConnection = mock(Connection.class); when(mockConnection.getHost()).thenReturn(host); HostConnectionPool<Integer> mockPool = mock(HostConnectionPool.class); when(mockPool.isActive()).thenAnswer(new Answer<Boolean>() { @Override public Boolean answer(InvocationOnMock invocation) throws Throwable { return status.get(); } }); when(mockPool.borrowConnection(any(Integer.class), any(TimeUnit.class))).thenReturn(mockConnection); when(mockPool.getHost()).thenReturn(host); when(mockConnection.getParentConnectionPool()).thenReturn(mockPool); return mockPool; } /** cqlsh:dyno_bootstrap> select "availabilityZone","hostname","token" from tokens where "appId" = 'dynomite_redis_puneet'; availabilityZone | hostname | token ------------------+--------------------------------------------+------------ us-east-1c | ec2-54-83-179-213.compute-1.amazonaws.com | 1383429731 us-east-1c | ec2-54-224-184-99.compute-1.amazonaws.com | 309687905 us-east-1c | ec2-54-91-190-159.compute-1.amazonaws.com | 3530913377 us-east-1c | ec2-54-81-31-218.compute-1.amazonaws.com | 2457171554 us-east-1e | ec2-54-198-222-153.compute-1.amazonaws.com | 309687905 us-east-1e | ec2-54-198-239-231.compute-1.amazonaws.com | 2457171554 us-east-1e | ec2-54-226-212-40.compute-1.amazonaws.com | 1383429731 us-east-1e | ec2-54-197-178-229.compute-1.amazonaws.com | 3530913377 cqlsh:dyno_bootstrap> */ private TokenMapSupplier getTokenMapSupplier() { final Map<Host, HostToken> tokenMap = new HashMap<Host, HostToken>(); tokenMap.put(h1, new HostToken(1383429731L, h1)); tokenMap.put(h2, new HostToken(3530913377L, h2)); tokenMap.put(h3, new HostToken(1383429731L, h3)); tokenMap.put(h4, new HostToken(3530913377L, h4)); tokenMap.put(h5, new HostToken(1383429731L, h5)); tokenMap.put(h6, new HostToken(3530913377L, h6)); return new TokenMapSupplier() { @Override public List<HostToken> getTokens(Set<Host> activeHosts) { return new ArrayList<HostToken>(tokenMap.values()); } @Override public HostToken getTokenForHost(Host host, Set<Host> activeHosts) { return tokenMap.get(host); } }; } private HashPartitioner getMockHashPartitioner(final Long hash) { return new HashPartitioner() { @Override public Long hash(int key) { return hash; } @Override public Long hash(long key) { return hash; } @Override public Long hash(String key) { return hash; } @Override public Long hash(byte[] key) { return hash; } @Override public HostToken getToken(Long keyHash) { throw new RuntimeException("NotImplemented"); } }; } }