package com.bazaarvoice.curator.recipes.leader; import com.google.common.util.concurrent.AbstractExecutionThreadService; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.Service; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.leader.LeaderLatch; import org.apache.curator.framework.recipes.leader.Participant; import org.apache.curator.framework.state.ConnectionState; import org.apache.curator.framework.state.ConnectionStateListener; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.EOFException; import java.io.IOException; import java.util.Collection; import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; /** * Starts and stops a Guava service based on whether this process is elected leader using a Curator leadership * election algorithm. * <p> * Because Guava services are not restartable, the {@code LeaderService} requires a {@code Supplier<Service>} * that it will call each time leadership is acquired. The {@code LeaderService} will ensure that only one * such {@code Service} instance is running at a time. * </p> * <p> * A typical use of {@code LeaderService} for a task that polls an external server might look like this: * </p> * <pre> * CuratorFramework curator = ...; * String serverId = new HostAndPort(InetAddress.getLocalHost().getHostAddress(), "8080").toString(); * final Duration pollInterval = ...; * * new LeaderService(curator, "/applications/my-app/leader", serverId, 1, TimeUnit.MINUTES, new Supplier<Service>() { * @Override * public Service get() { * return new AbstractScheduledService() { * @Override * protected void runOneIteration() throws Exception { * poll(); // TODO: implement * } * * @Override * protected Scheduler scheduler() { * return Scheduler.newFixedDelaySchedule(0, duration.toMillis(), TimeUnit.MILLISECONDS); * } * }; * } * }).start(); * </pre> */ public class LeaderService extends AbstractExecutionThreadService { private static final Logger LOG = LoggerFactory.getLogger(LeaderService.class); private final ConnectionStateListener _listener = new ConnectionStateListener() { @Override public void stateChanged(CuratorFramework curatorFramework, ConnectionState newState) { if (newState == ConnectionState.LOST || newState == ConnectionState.SUSPENDED) { LOG.debug("Lost leadership due to ZK state change to {}: {}", newState, getId()); closeLeaderLatch(); } } }; private final CuratorFramework _curator; private final String _leaderPath; private final String _instanceId; private final String _serviceName; private final Supplier<Service> _serviceFactory; private final long _reacquireDelayNanos; private volatile LeaderLatch _latch; private volatile Service _delegate; /** * Creates an instance of the service that will create, start and stop a managed delegate service as this * instance acquires and loses leadership in a leadership election. * @param curator A started connection to ZooKeeper. * @param leaderPath The ZooKeeper path under which the leadership election algorithm will create ephemeral * ZooKeeper nodes. * @param instanceId An identifier for this instance, included in the information returned by the * {@link #getLeader()} and {@link #getParticipants()} instances. * @param serviceName The name of this service. This will be used to name a Java thread dedicated to the * leadership election algorithm for this instance. * @param reacquireDelay The amount of time to wait before attempting to re-acquire leadership after losing * leadership due to ZooKeeper connection loss or after relinquishing leadership due to * another process manually stopping the managed delegate service. It may be desirable * to set this to a relatively high value for services that are expensive to start to * avoid a rapid sequence of restarts in the presence of network issues that cause the * connection to ZooKeeper to flap back and forth. * @param reacquireDelayUnit The unit of the <code>reacquireDelay</code> argument. * @param serviceFactory A factory for delegate service instances. This factory will be used to create a new * instance of the delegate service each time leadership is acquired. Because Guava * services cannot be restarted, a new instance of the delegate service must be created * each time leadership is acquired. */ public LeaderService(CuratorFramework curator, String leaderPath, String instanceId, String serviceName, long reacquireDelay, TimeUnit reacquireDelayUnit, Supplier<Service> serviceFactory) { _curator = Objects.requireNonNull(curator, "curator"); _leaderPath = Objects.requireNonNull(leaderPath, "leaderPath"); _instanceId = Objects.requireNonNull(instanceId, "instanceId"); _serviceName = Objects.requireNonNull(serviceName, "serviceName"); _serviceFactory = Objects.requireNonNull(serviceFactory, "serviceFactory"); _reacquireDelayNanos = Objects.requireNonNull(reacquireDelayUnit, "reacquireDelayUnit").toNanos(reacquireDelay); checkArgument(_reacquireDelayNanos >= 0, "reacquireDelay must be non-negative"); initLeaderLatch(); } @Override protected String serviceName() { return _serviceName; } /** * @return This instance's participant id provided at construction time. This will be the value returned * when {@link #getParticipants()} is called. */ public String getId() { return _instanceId; } /** * <B>NOTE</B> - this method polls the ZooKeeper server. Therefore it may return a value that does not match * {@link #hasLeadership()} as hasLeadership returns a cached value. * * @return The set of current participants in the leader selection. * @throws Exception ZooKeeper or thread error. */ public Collection<Participant> getParticipants() throws Exception { return _latch.getParticipants(); } /** * <B>NOTE</B> - this method polls the ZooKeeper server. Therefore it may return a value that does not match * {@link #hasLeadership()} as hasLeadership returns a cached value. * * @return The id for the current leader. If for some reason there is no current leader, a dummy participant * is returned. * @throws Exception ZooKeeper or thread error. */ public Participant getLeader() throws Exception { return _latch.getLeader(); } /** @return True if leadership is currently held by this instance. */ public boolean hasLeadership() { return _latch.hasLeadership(); } /** * @return The current wrapped service instance, if any. Returns {@link Optional#empty()} ()} when this instance * does not own the leadership lock. */ public Optional<Service> getCurrentDelegateService() { return _latch.hasLeadership() ? Optional.ofNullable(_delegate) : Optional.<Service>empty(); } @Override protected void startUp() throws Exception { _curator.getConnectionStateListenable().addListener(_listener); } @Override protected void shutDown() throws Exception { _curator.getConnectionStateListenable().removeListener(_listener); } @Override protected void triggerShutdown() { // Release leadership (if we have it) and wake up the main execution thread (if it's sleeping). closeLeaderLatch(); } @Override protected void run() throws InterruptedException { // Beware race conditions: closeLeaderLatch() may be called by another thread at any time. while (isRunning()) { try { // Start attempting to acquire leadership via the Curator leadership latch. LOG.debug("Attempting to acquire leadership: {}", getId()); LeaderLatch latch = startLeaderLatch(); // Wait until (a) leadership is acquired or (b) the latch is closed by service shutdown or ZK cxn loss. if (isRunning()) { try { latch.await(); } catch (EOFException e) { // Latch was closed while we were waiting. checkState(!latch.hasLeadership()); } } // If we succeeded in acquiring leadership, start/run the leadership-managed delegate service. if (isRunning() && latch.hasLeadership()) { LOG.debug("Leadership acquired: {}", getId()); runAsLeader(); LOG.debug("Leadership released: {}", getId()); } } finally { closeLeaderLatch(); } if (isRunning()) { // If we lost or relinquished leadership, wait a while for things to settle before trying to // re-acquire leadership (eg. wait for a network hiccup to the ZooKeeper server to resolve). sleep(_reacquireDelayNanos); } } } private void runAsLeader() throws InterruptedException { try { _delegate = listenTo(_serviceFactory.get()); _delegate.startAsync().awaitRunning(); try { awaitLeadershipLostOrServicesStopped(); } finally { _delegate.stopAsync().awaitTerminated(); } } catch (InterruptedException ie) { throw ie; } catch (Throwable t) { // Start may have failed due to a network error, we'll sleep for reacquireDelay and try again. LOG.error("Exception starting or stopping leadership-managed service: {}", getId(), t); } finally { _delegate = null; } } private LeaderLatch newLeaderLatch() { return new LeaderLatch(_curator, _leaderPath, _instanceId); } // IMPORTANT: ************************************************************************************************** // All updates to the '_latch' object are synchronized to avoid race conditions between the service execution // thread and the various listeners (ZK connection state listener, delegate service listener). private synchronized void initLeaderLatch() { // Create a non-started latch that we can use to implement getLeader(), getParticipants() & friends. _latch = newLeaderLatch(); } private synchronized LeaderLatch startLeaderLatch() throws InterruptedException { LeaderLatch latch = _latch; // Read the volatile once // Assert not started already. initLeaderLatch() and closeLeaderLatch() leave the latch in the latent state. checkState(latch.getState() == LeaderLatch.State.LATENT); try { latch.start(); } catch (InterruptedException ie) { throw ie; } catch (Throwable t) { LOG.error("Exception attempting to acquire leadership: {}", getId(), t); } return latch; } private synchronized void closeLeaderLatch() { LeaderLatch latch = _latch; // Read the volatile once if (latch.getState() == LeaderLatch.State.STARTED) { try { latch.close(); } catch (IOException e) { LOG.debug("Unexpected exception closing LeaderLatch.", e); } } // Return the latch to a latent state (newly created) for use by getLeader(), getParticipants() & friends. _latch = newLeaderLatch(); // Wake up the main execution thread. notifyAll(); } /** Wait until we lose leadership or this service is stopped or the delegate service is stopped. */ private synchronized void awaitLeadershipLostOrServicesStopped() throws InterruptedException { while (_latch.hasLeadership() && isRunning() && _delegate.isRunning()) { wait(); } } /** Wait for the specified amount of time or until this service is stopped, whichever comes first. */ private synchronized void sleep(long waitNanos) throws InterruptedException { while (waitNanos > 0 && isRunning()) { long start = System.nanoTime(); TimeUnit.NANOSECONDS.timedWait(this, waitNanos); waitNanos -= System.nanoTime() - start; } } /** Release leadership when the service terminates (normally or abnormally). */ private Service listenTo(Service delegate) { delegate.addListener(new Listener() { @Override public void starting() { // Do nothing } @Override public void running() { // Do nothing } @Override public void stopping(State from) { // Do nothing } @Override public void terminated(State from) { closeLeaderLatch(); } @Override public void failed(State from, Throwable failure) { closeLeaderLatch(); } }, MoreExecutors.directExecutor()); return delegate; } }