com.netflix.loadbalancer.ServerStats Java Examples

private List<Double> doCalculateTotalWeights(List<ServiceCombServer> servers) {
  List<Double> stats = new ArrayList<>(servers.size() + 1);
  double totalWeights = 0;
  boolean needRandom = false;
  for (ServiceCombServer server : servers) {
    ServerStats serverStats = loadBalancer.getLoadBalancerStats().getSingleServerStat(server);
    double avgTime = serverStats.getResponseTimeAvg();
    if (!needRandom && avgTime > MIN_GAP) {
      needRandom = true;
    }
    totalWeights += avgTime;
    stats.add(avgTime);
  }
  stats.add(totalWeights);
  totalWeightsCache = totalWeights;
  if (needRandom) {
    return stats;
  } else {
    return new ArrayList<>();
  }
}
 

public PooledConnection(final Channel channel, final Server server, final ClientChannelManager channelManager,
                 final InstanceInfo serverKey,
                 final ServerStats serverStats, 
                 final Counter closeConnCounter, 
                 final Counter closeWrtBusyConnCounter)
{
    this.channel = channel;
    this.server = server;
    this.channelManager = channelManager;
    this.serverKey = serverKey;
    this.serverStats = serverStats;
    this.creationTS = System.currentTimeMillis();
    this.closeConnCounter = closeConnCounter;
    this.closeWrtBusyConnCounter = closeWrtBusyConnCounter;
    
    this.connectionState = ConnectionState.WRITE_READY;

    // Store ourself as an attribute on the underlying Channel.
    channel.attr(CHANNEL_ATTR).set(this);
}
 

private boolean isErrorThresholdMet() {
  LoadBalancerStats stats = loadBalancer.getLoadBalancerStats();

  if (stats != null && stats.getServerStats() != null && stats.getServerStats().size() > 0) {
    ServerStats serverStats = stats.getSingleServerStat(lastServer);
    int successiveFaildCount = serverStats.getSuccessiveConnectionFailureCount();
    if (Configuration.INSTANCE.getSuccessiveFailedTimes(microserviceName) > 0
        && successiveFaildCount >= Configuration.INSTANCE.getSuccessiveFailedTimes(microserviceName)) {
      serverStats.clearSuccessiveConnectionFailureCount();
      return true;
    }
  }
  return false;
}
 

public PerServerConnectionPool(
        Server server,
        ServerStats stats,
        InstanceInfo instanceInfo,
        SocketAddress serverAddr,
        NettyClientConnectionFactory connectionFactory,
        PooledConnectionFactory pooledConnectionFactory,
        ConnectionPoolConfig config,
        IClientConfig niwsClientConfig,
        Counter createNewConnCounter,
        Counter createConnSucceededCounter,
        Counter createConnFailedCounter,
        Counter requestConnCounter, Counter reuseConnCounter,
        Counter connTakenFromPoolIsNotOpen,
        Counter maxConnsPerHostExceededCounter,
        Timer connEstablishTimer,
        AtomicInteger connsInPool,
        AtomicInteger connsInUse) {
    this.server = server;
    this.stats = stats;
    this.instanceInfo = instanceInfo;
    // Note: child classes can sometimes connect to different addresses than
    this.serverAddr = Objects.requireNonNull(serverAddr, "serverAddr");
    this.connectionFactory = connectionFactory;
    this.pooledConnectionFactory = pooledConnectionFactory;
    this.config = config;
    this.niwsClientConfig = niwsClientConfig;
    this.createNewConnCounter = createNewConnCounter;
    this.createConnSucceededCounter = createConnSucceededCounter;
    this.createConnFailedCounter = createConnFailedCounter;
    this.requestConnCounter = requestConnCounter;
    this.reuseConnCounter = reuseConnCounter;
    this.connTakenFromPoolIsNotOpen = connTakenFromPoolIsNotOpen;
    this.maxConnsPerHostExceededCounter = maxConnsPerHostExceededCounter;
    this.connEstablishTimer = connEstablishTimer;
    this.connsInPool = connsInPool;
    this.connsInUse = connsInUse;
    
    this.connCreationsInProgress = new AtomicInteger(0);
}
 

protected IConnectionPool createConnectionPool(
        Server chosenServer, ServerStats stats, InstanceInfo instanceInfo, SocketAddress serverAddr,
        NettyClientConnectionFactory clientConnFactory, PooledConnectionFactory pcf,
        ConnectionPoolConfig connPoolConfig, IClientConfig clientConfig, Counter createNewConnCounter,
        Counter createConnSucceededCounter, Counter createConnFailedCounter, Counter requestConnCounter,
        Counter reuseConnCounter, Counter connTakenFromPoolIsNotOpen, Counter maxConnsPerHostExceededCounter,
        PercentileTimer connEstablishTimer, AtomicInteger connsInPool, AtomicInteger connsInUse) {
    return new PerServerConnectionPool(
            chosenServer,
            stats,
            instanceInfo,
            serverAddr,
            clientConnFactory,
            pcf,
            connPoolConfig,
            clientConfig,
            createNewConnCounter,
            createConnSucceededCounter,
            createConnFailedCounter,
            requestConnCounter,
            reuseConnCounter,
            connTakenFromPoolIsNotOpen,
            maxConnsPerHostExceededCounter,
            connEstablishTimer,
            connsInPool,
            connsInUse
    );
}
 

@Test
public void testObservableWithMultipleServersFailed() throws Exception {        
    IClientConfig config = IClientConfig.Builder.newBuilder()
            .withDefaultValues()
            .withRetryOnAllOperations(true)
            .withMaxAutoRetries(1)
            .withMaxAutoRetriesNextServer(3)
            .withConnectTimeout(100)
            .build();
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createGet("/testAsync/person");
    BaseLoadBalancer lb = new BaseLoadBalancer(new DummyPing(), new AvailabilityFilteringRule());        
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config);
    Server badServer = new Server("localhost:12345");
    Server badServer1 = new Server("localhost:12346");
    Server badServer2 = new Server("localhost:12347");

    List<Server> servers = Lists.newArrayList(badServer, badServer1, badServer2);
    lb.setServersList(servers);
    Observable<Person> observableWithRetries = getPersonObservable(lbObservables.submit(request));
    ObserverWithLatch<Person> observer = new ObserverWithLatch<Person>();
    observableWithRetries.subscribe(observer);
    observer.await();
    assertNull(observer.obj);
    observer.error.printStackTrace();
    assertTrue(observer.error instanceof ClientException);
    
    ServerStats stats = lbObservables.getServerStats(badServer);
    // two requests to bad server because retry same server is set to 1
    assertEquals(2, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(2, stats.getSuccessiveConnectionFailureCount());
}
 

@Test
public void testObservableWithRetrySameServer() throws Exception {
    IClientConfig config = DefaultClientConfigImpl.getClientConfigWithDefaultValues().withProperty(CommonClientConfigKey.ConnectTimeout, "1000");
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createGet("/testAsync/person");
    Server badServer = new Server("localhost:12345");
    Server goodServer = new Server("localhost:" + port);
    List<Server> servers = Lists.newArrayList(badServer, badServer, goodServer);
    
    BaseLoadBalancer lb = LoadBalancerBuilder.<Server>newBuilder()
            .withRule(new AvailabilityFilteringRule())
            .withPing(new DummyPing())
            .buildFixedServerListLoadBalancer(servers);
    
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config,
            new NettyHttpLoadBalancerErrorHandler(1, 0, true));

    Observable<Person> observableWithRetries = getPersonObservable(lbObservables.submit(request));
    ObserverWithLatch<Person> observer = new ObserverWithLatch<Person>();
    observableWithRetries.subscribe(observer);
    observer.await();
    assertNull(observer.obj);
    assertTrue(observer.error instanceof ClientException);

    ServerStats stats = lbObservables.getServerStats(badServer);
    
    // two requests to bad server because retry same server is set to 1
    assertEquals(2, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    
    stats = lbObservables.getServerStats(goodServer);
    assertEquals(0, stats.getTotalRequestsCount());
}
 

@Test
public void testObservableWithMultipleServersWithOverrideRxConfig() throws Exception {
    IClientConfig config = DefaultClientConfigImpl.getClientConfigWithDefaultValues().withProperty(CommonClientConfigKey.ConnectTimeout, "1000");
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createGet("/testAsync/person");
    Server badServer = new Server("localhost:12345");
    Server goodServer = new Server("localhost:" + port);
    List<Server> servers = Lists.newArrayList(badServer, badServer, badServer, goodServer);
    
    BaseLoadBalancer lb = LoadBalancerBuilder.<Server>newBuilder()
            .withRule(new AvailabilityFilteringRule())
            .withPing(new DummyPing())
            .buildFixedServerListLoadBalancer(servers);
    
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config,
            new NettyHttpLoadBalancerErrorHandler(1, 3, true));
    HttpClientConfig rxconfig = HttpClientConfig.Builder.newDefaultConfig();
    Person person = getPersonObservable(lbObservables.submit(request, rxconfig)).toBlocking().single();
    assertEquals(EmbeddedResources.defaultPerson, person);
    ServerStats stats = lbObservables.getServerStats(badServer);
    // two requests to bad server because retry same server is set to 1
    assertEquals(4, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(4, stats.getSuccessiveConnectionFailureCount());
    
    stats = lbObservables.getServerStats(goodServer);
    assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(0, stats.getSuccessiveConnectionFailureCount());
    
    final HttpClientListener listener = lbObservables.getListener();
    assertEquals(1, listener.getConnectionCount());
    waitUntilTrueOrTimeout(1000, new Func0<Boolean>() {
        @Override
        public Boolean call() {
            return listener.getPoolReleases() == 1;
        }
    });
}
 

@Test
public void testObservableWithMultipleServers() throws Exception {
    IClientConfig config = DefaultClientConfigImpl
            .getClientConfigWithDefaultValues()
            .withProperty(CommonClientConfigKey.ConnectTimeout, "1000");
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createGet("/testAsync/person");
    Server badServer = new Server("localhost:12345");
    Server goodServer = new Server("localhost:" + port);
    List<Server> servers = Lists.newArrayList(badServer, badServer, badServer, goodServer);
    
    BaseLoadBalancer lb = LoadBalancerBuilder.<Server>newBuilder()
            .withRule(new AvailabilityFilteringRule())
            .withPing(new DummyPing())
            .buildFixedServerListLoadBalancer(servers);
    
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config,
            new NettyHttpLoadBalancerErrorHandler(1, 3, true));
    Person person = getPersonObservable(lbObservables.submit(request)).toBlocking().single();
    assertEquals(EmbeddedResources.defaultPerson, person);
    ServerStats stats = lbObservables.getServerStats(badServer);
    // two requests to bad server because retry same server is set to 1
    assertEquals(4, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(4, stats.getSuccessiveConnectionFailureCount());
    
    stats = lbObservables.getServerStats(goodServer);
    assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(0, stats.getSuccessiveConnectionFailureCount());

    person = getPersonObservable(lbObservables.submit(request)).toBlocking().single();
    assertEquals(EmbeddedResources.defaultPerson, person);
    HttpClientListener listener = lbObservables.getListener();
    assertEquals(1, listener.getPoolReuse());
}
 

@Test
public void testSuccessfulRetries() throws Exception {
    lb.setServersList(Lists.newArrayList(new Server("localhost:12987"), new Server("localhost:12987"), localServer));
    URI localUrl = new URI("/ok");
    HttpRequest request = HttpRequest.newBuilder().uri(localUrl).queryParams("name", "ribbon").build();
    try {
        HttpResponse response = client.executeWithLoadBalancer(request, DefaultClientConfigImpl.getEmptyConfig().set(CommonClientConfigKey.MaxAutoRetriesNextServer, 2));
        assertEquals(200, response.getStatus());
    } catch (ClientException e) { 
        fail("Unexpected exception");
    }
    ServerStats stats = lb.getLoadBalancerStats().getSingleServerStat(new Server("localhost:12987")); 
    assertEquals(1, stats.getSuccessiveConnectionFailureCount());
}
 

@Test
public void testRetriesOnPostWithConnectException() throws Exception {
    URI localUrl = new URI("/status?code=503");
    lb.setServersList(Lists.newArrayList(localServer));
    HttpRequest request = HttpRequest.newBuilder().uri(localUrl).verb(Verb.POST).setRetriable(true).build();
    try {
        HttpResponse response = client.executeWithLoadBalancer(request, DefaultClientConfigImpl.getEmptyConfig().set(CommonClientConfigKey.MaxAutoRetriesNextServer, 2));
        fail("Exception expected");
    } catch (ClientException e) { // NOPMD
    }
    ServerStats stats = lb.getLoadBalancerStats().getSingleServerStat(localServer); 
    assertEquals(3, stats.getSuccessiveConnectionFailureCount());
}
 

@Test
public void testRetriesOnPost() throws Exception {
    URI localUrl = new URI("/noresponse");
    HttpRequest request = HttpRequest.newBuilder().uri(localUrl).verb(Verb.POST).setRetriable(true).build();
    try {
        client.executeWithLoadBalancer(request, DefaultClientConfigImpl.getEmptyConfig().set(CommonClientConfigKey.MaxAutoRetriesNextServer, 2));
        fail("Exception expected");
    } catch (ClientException e) { // NOPMD
    }
    ServerStats stats = lb.getLoadBalancerStats().getSingleServerStat(localServer); 
    assertEquals(3, stats.getSuccessiveConnectionFailureCount());
}
 

@Test
public void postReadTimeout() throws Exception {
    URI localUrl = new URI("/noresponse");
    HttpRequest request = HttpRequest.newBuilder().uri(localUrl).verb(Verb.POST).build();
    try {
        client.executeWithLoadBalancer(request, DefaultClientConfigImpl.getEmptyConfig().set(CommonClientConfigKey.MaxAutoRetriesNextServer, 2));
        fail("Exception expected");
    } catch (ClientException e) { // NOPMD
    }
    ServerStats stats = lb.getLoadBalancerStats().getSingleServerStat(localServer); 
    assertEquals(1, stats.getSuccessiveConnectionFailureCount());
}
 

@VisibleForTesting
ServerStats getServerStats(Server server) {
    return lbContext.getServerStats(server);
}
 

@Test
public void testLoadBalancingObservablesWithReadTimeout() throws Exception {
    NettyHttpLoadBalancerErrorHandler errorHandler = new NettyHttpLoadBalancerErrorHandler(1, 3, true);
    MockWebServer server = new MockWebServer();
    String content = "{\"name\": \"ribbon\", \"age\": 2}";
    server.enqueue(new MockResponse().setResponseCode(200).setHeader("Content-type", "application/json")
            .setBody(content));       
    server.play();

    IClientConfig config = DefaultClientConfigImpl
            .getClientConfigWithDefaultValues()
            .set(CommonClientConfigKey.ReadTimeout, 100);
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createGet("/testAsync/readTimeout");
    
    BaseLoadBalancer lb = new BaseLoadBalancer(new DummyPing(), new AvailabilityFilteringRule());
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config, errorHandler);
    
    Server goodServer = new Server("localhost:" + server.getPort());
    Server badServer  = new Server("localhost:" + port);
    lb.setServersList(Lists.newArrayList(goodServer, badServer, badServer, goodServer));
    
    Observable<Person> observableWithRetries = getPersonObservable(lbObservables.submit(request));
    ObserverWithLatch<Person> observer = new ObserverWithLatch<Person>();
    observableWithRetries.subscribe(observer);
    observer.await();
    if (observer.error != null) {
        observer.error.printStackTrace();
    }
    assertEquals("ribbon", observer.obj.name);
    assertEquals(2, observer.obj.age);
    ServerStats stats = lbObservables.getServerStats(badServer);
    server.shutdown();
    
    final HttpClientListener listener = lbObservables.getListener();
    waitUntilTrueOrTimeout(1000, new Func0<Boolean>() {
        @Override
        public Boolean call() {
            return listener.getPoolReleases() == 5;
        }
    });
    assertEquals(0, listener.getPoolReuse());
    
    // two requests to bad server because retry same server is set to 1
    assertEquals(4, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(4, stats.getSuccessiveConnectionFailureCount());
    
    stats = lbObservables.getServerStats(goodServer);
    assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(0, stats.getSuccessiveConnectionFailureCount());
}
 

@Test
public void testLoadBalancingWithTwoServers() throws Exception {
    MockWebServer server = new MockWebServer();
    String content = "{\"name\": \"ribbon\", \"age\": 2}";
    server.enqueue(new MockResponse().setResponseCode(200).setHeader("Content-type", "application/json")
            .setBody(content));       
    server.play();

    IClientConfig config = DefaultClientConfigImpl.getClientConfigWithDefaultValues();
            
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createPost("/testAsync/person")
            .withContent(SerializationUtils.serializeToBytes(JacksonCodec.getInstance(), EmbeddedResources.defaultPerson, null))
            .withHeader("Content-type", "application/json");
    NettyHttpLoadBalancerErrorHandler errorHandler = new NettyHttpLoadBalancerErrorHandler(1, 3, true);
    BaseLoadBalancer lb = new BaseLoadBalancer(new DummyPing(), new AvailabilityFilteringRule());
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config, errorHandler);
    HttpClientListener externalListener = HttpClientListener.newHttpListener("external");
    lbObservables.subscribe(externalListener);
    Server server1 = new Server("localhost:" + server.getPort());
    Server server2 = new Server("localhost:" + port);
    
    lb.setServersList(Lists.newArrayList(server1, server2));
    RetryHandler handler = new RequestSpecificRetryHandler(true, true, errorHandler, null) {
        @Override
        public boolean isRetriableException(Throwable e, boolean sameServer) {
            return true;
        }
    };
    Observable<Person> observableWithRetries = getPersonObservable(lbObservables.submit(request, handler, null));
    ObserverWithLatch<Person> observer = new ObserverWithLatch<Person>();
    observableWithRetries.subscribe(observer);
    observer.await();
    if (observer.error != null) {
        observer.error.printStackTrace();
    }
    assertEquals("ribbon", observer.obj.name);
    assertEquals(EmbeddedResources.defaultPerson.age, observer.obj.age);
    
    observer = new ObserverWithLatch<Person>();
    observableWithRetries = getPersonObservable(lbObservables.submit(request, handler, null));
    observableWithRetries.subscribe(observer);
    observer.await();
    if (observer.error != null) {
        observer.error.printStackTrace();
    }
    assertEquals("ribbon", observer.obj.name);
    assertEquals(2, observer.obj.age);
    
    ServerStats stats = lbObservables.getServerStats(server1);
    server.shutdown();
    // assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    
    stats = lbObservables.getServerStats(server2);
    // two requests to bad server because retry same server is set to 1
    assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(0, stats.getSuccessiveConnectionFailureCount());
    final HttpClientListener listener = lbObservables.getListener();
    assertEquals(2, listener.getPoolAcquires());
    waitUntilTrueOrTimeout(1000, new Func0<Boolean>() {
        @Override
        public Boolean call() {
            return listener.getPoolReleases() == 2;
        }
    });
    assertEquals(2, listener.getConnectionCount());
    assertEquals(0, listener.getPoolReuse());
    assertEquals(2, externalListener.getPoolAcquires());
}
 

@Test
public void testLoadBalancingPostWithReadTimeout() throws Exception {
    MockWebServer server = new MockWebServer();
    String content = "{\"name\": \"ribbon\", \"age\": 2}";
    server.enqueue(new MockResponse()
            .setResponseCode(200)
            .setHeader("Content-type", "application/json")
            .setBody(content));       
    server.play();

    IClientConfig config = DefaultClientConfigImpl
            .getClientConfigWithDefaultValues()
            .set(CommonClientConfigKey.ReadTimeout, 100);
    
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createPost("/testAsync/postTimeout")
            .withContent(SerializationUtils.serializeToBytes(JacksonCodec.getInstance(), EmbeddedResources.defaultPerson, null))
            .withHeader("Content-type", "application/json");
    
    NettyHttpLoadBalancerErrorHandler errorHandler = new NettyHttpLoadBalancerErrorHandler(1, 3, true);
    BaseLoadBalancer lb = new BaseLoadBalancer(new DummyPing(), new AvailabilityFilteringRule());
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config, errorHandler);
    Server goodServer = new Server("localhost:" + server.getPort());
    Server badServer = new Server("localhost:" + port);
    List<Server> servers = Lists.newArrayList(badServer, badServer, badServer, goodServer);
    lb.setServersList(servers);
    RetryHandler handler = new RequestSpecificRetryHandler(true, true, errorHandler, null) {
        @Override
        public boolean isRetriableException(Throwable e, boolean sameServer) {
            return true;
        }
    };
    Observable<Person> observableWithRetries = getPersonObservable(lbObservables.submit(request, handler, null));
    ObserverWithLatch<Person> observer = new ObserverWithLatch<Person>();
    observableWithRetries.subscribe(observer);
    observer.await();
    if (observer.error != null) {
        observer.error.printStackTrace();
    }
    assertEquals("ribbon", observer.obj.name);
    assertEquals(2, observer.obj.age);
    ServerStats stats = lbObservables.getServerStats(badServer);
    server.shutdown();
    assertEquals(4, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(4, stats.getSuccessiveConnectionFailureCount());
    
    stats = lbObservables.getServerStats(goodServer);
    // two requests to bad server because retry same server is set to 1
    assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(0, stats.getSuccessiveConnectionFailureCount());
}
 

@Test
public void testLoadBalancingPostWithNoRetrySameServer() throws Exception {
    MockWebServer server = new MockWebServer();
    String content = "{\"name\": \"ribbon\", \"age\": 2}";
    server.enqueue(new MockResponse().setResponseCode(200).setHeader("Content-type", "application/json")
            .setBody(content));       
    server.play();

    IClientConfig config = DefaultClientConfigImpl.getClientConfigWithDefaultValues()
            .set(CommonClientConfigKey.ReadTimeout, 100);
    HttpClientRequest<ByteBuf> request = HttpClientRequest.createPost("/testAsync/postTimeout")
            .withContent(SerializationUtils.serializeToBytes(JacksonCodec.getInstance(), EmbeddedResources.defaultPerson, null))
            .withHeader("Content-type", "application/json");
    NettyHttpLoadBalancerErrorHandler errorHandler = new NettyHttpLoadBalancerErrorHandler(0, 3, true);
    BaseLoadBalancer lb = new BaseLoadBalancer(new DummyPing(), new AvailabilityFilteringRule());
    LoadBalancingHttpClient<ByteBuf, ByteBuf> lbObservables = RibbonTransport.newHttpClient(lb, config, errorHandler);
    Server goodServer = new Server("localhost:" + server.getPort());
    Server badServer = new Server("localhost:" + port);
    List<Server> servers = Lists.newArrayList(badServer, badServer, badServer, goodServer);
    lb.setServersList(servers);
    RetryHandler handler = new RequestSpecificRetryHandler(true, true, errorHandler, null) {
        @Override
        public boolean isRetriableException(Throwable e, boolean sameServer) {
            return true;
        }
    };
    Observable<Person> observableWithRetries = getPersonObservable(lbObservables.submit(request, handler, null));
    ObserverWithLatch<Person> observer = new ObserverWithLatch<Person>();
    observableWithRetries.subscribe(observer);
    observer.await();
    if (observer.error != null) {
        observer.error.printStackTrace();
    }
    server.shutdown();
    assertEquals("ribbon", observer.obj.name);
    assertEquals(2, observer.obj.age);
    ServerStats stats = lbObservables.getServerStats(badServer);
    assertEquals(2, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(2, stats.getSuccessiveConnectionFailureCount());
    
    stats = lbObservables.getServerStats(goodServer);
    assertEquals(1, stats.getTotalRequestsCount());
    assertEquals(0, stats.getActiveRequestsCount());
    assertEquals(0, stats.getSuccessiveConnectionFailureCount());
}
 

public ServerStats getServerStats() {
    return serverStats;
}
 

public ChannelFuture close() {
    final ServerStats stats = getServerStats();
    stats.decrementOpenConnectionsCount();
    closeConnCounter.increment();
    return channel.close();
}
 

public void updateServerStats() {
    final ServerStats stats = getServerStats();
    stats.decrementOpenConnectionsCount();
    stats.close();
}
 

@Override
public boolean release(final PooledConnection conn) {

    conn.stopRequestTimer();
    releaseConnCounter.increment();
    connsInUse.decrementAndGet();

    final ServerStats stats = conn.getServerStats();
    stats.decrementActiveRequestsCount();
    stats.incrementNumRequests();

    if (shuttingDown) {
        return false;
    }

    boolean released = false;

    if (conn.isShouldClose() ||
            // if the connection has been around too long (i.e. too many requests), then close it
            conn.getUsageCount() > connPoolConfig.getMaxRequestsPerConnection()) {

        // Close and discard the connection, as it has been flagged (possibly due to receiving a non-channel error like a 503).
        conn.setInPool(false);
        conn.close();
    }
    else if (stats.isCircuitBreakerTripped()) {
        // Don't put conns for currently circuit-tripped servers back into the pool.
        conn.setInPool(false);
        conn.close();
    }
    else if (!conn.isActive()) {
        // Connection is already closed, so discard.
        alreadyClosedCounter.increment();
        // make sure to decrement OpenConnectionCounts
        conn.updateServerStats();
        conn.setInPool(false);
    }
    else {
        releaseHandlers(conn);

        // Attempt to return connection to the pool.
        IConnectionPool pool = perServerPools.get(conn.getServer());
        if (pool != null) {
            released = pool.release(conn);
        }
        else {
            // The pool for this server no longer exists (maybe due to it failling out of
            // discovery).
            conn.setInPool(false);
            released = false;
            conn.close();
        }

        if (LOG.isDebugEnabled()) LOG.debug("PooledConnection released: " + conn.toString());
    }

    return released;
}
 

protected PooledConnectionFactory createPooledConnectionFactory(Server chosenServer, InstanceInfo instanceInfo, ServerStats stats, ClientChannelManager clientChannelMgr,
                                                                Counter closeConnCounter, Counter closeWrtBusyConnCounter) {
    return ch -> new PooledConnection(ch, chosenServer, clientChannelMgr, instanceInfo, stats, closeConnCounter, closeWrtBusyConnCounter);
}