Java Code Examples for org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService#start()

/**
 * Tests that the standalone leader retrieval services return the specified address and the
 * fixed leader session id.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
	LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);
	rmLeaderRetrievalService.start(rmListener);

	verify(jmListener1).notifyLeaderAddress(eq("UNKNOWN"), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq("UNKNOWN"), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

@Override
public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception {
	Preconditions.checkState(DefaultJobLeaderService.State.STARTED == state, "The service is currently not running.");

	LOG.info("Add job {} for job leader monitoring.", jobId);

	final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
		jobId,
		defaultTargetAddress);

	DefaultJobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId);

	final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener));

	if (oldEntry != null) {
		oldEntry.f0.stop();
		oldEntry.f1.stop();
	}

	leaderRetrievalService.start(jobManagerLeaderListener);
}
 

/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	JobID jobId = new JobID();
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Tests that the standalone leader retrieval services return the given address and the
 * fixed leader session id.
 */
@Test
public void testJobMasterLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	final String jobManagerAddress1 = "foobar";
	final String jobManagerAddress2 = "barfoo";
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2);

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * Retrieves the leader akka url and the current leader session ID. The values are stored in a
 * {@link LeaderConnectionInfo} instance.
 *
 * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection
 *                               information
 * @param timeout Timeout when to give up looking for the leader
 * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session
 * ID
 * @throws LeaderRetrievalException
 */
public static LeaderConnectionInfo retrieveLeaderConnectionInfo(
		LeaderRetrievalService leaderRetrievalService,
		FiniteDuration timeout
) throws LeaderRetrievalException {
	LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener();

	try {
		leaderRetrievalService.start(listener);

		Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture();

		return Await.result(connectionInfoFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader address and leader " +
				"session ID.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 

@Test
public void testClosingReportsToLeader() throws Exception {
	final Configuration flinkConfig = new Configuration();

	try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) {
		final LeaderElectionService elector = services.getResourceManagerLeaderElectionService();
		final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever();
		final LeaderContender contender = mockContender(elector);
		final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class);

		elector.start(contender);
		retrieval.start(listener);

		// wait until the contender has become the leader
		verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class));

		// now we can close the election service
		services.close();

		verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class));
	}
}
 

@Test
public void testClosingReportsToLeader() throws Exception {
	final Configuration flinkConfig = new Configuration();

	try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) {
		final LeaderElectionService elector = services.getResourceManagerLeaderElectionService();
		final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever();
		final LeaderContender contender = mockContender(elector);
		final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class);

		elector.start(contender);
		retrieval.start(listener);

		// wait until the contender has become the leader
		verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class));

		// now we can close the election service
		services.close();

		verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class));
	}
}
 

/**
 * Tests that the standalone leader retrieval services return the specified address and the
 * fixed leader session id.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
	LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);
	rmLeaderRetrievalService.start(rmListener);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * Tests the ResourceManager leader retrieval for a given job.
 */
@Test
public void testResourceManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever();

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	JobID jobId = new JobID();
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Tests that the standalone leader retrieval services return the specified address and the
 * fixed leader session id.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
	LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);
	rmLeaderRetrievalService.start(rmListener);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * Retrieves the leader akka url and the current leader session ID. The values are stored in a
 * {@link LeaderConnectionInfo} instance.
 *
 * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection
 *                               information
 * @param timeout Timeout when to give up looking for the leader
 * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session
 * ID
 * @throws LeaderRetrievalException
 */
public static LeaderConnectionInfo retrieveLeaderConnectionInfo(
		LeaderRetrievalService leaderRetrievalService,
		FiniteDuration timeout
) throws LeaderRetrievalException {
	LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener();

	try {
		leaderRetrievalService.start(listener);

		Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture();

		return Await.result(connectionInfoFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader address and leader " +
				"session ID.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 

/**
 * Retrieves the current leader gateway using the given {@link LeaderRetrievalService}. If the
 * current leader could not be retrieved after the given timeout, then a
 * {@link LeaderRetrievalException} is thrown.
 *
 * @param leaderRetrievalService {@link LeaderRetrievalService} which is used for the leader retrieval
 * @param actorSystem ActorSystem which is used for the {@link LeaderRetrievalListener} implementation
 * @param timeout Timeout value for the retrieval call
 * @return The current leader gateway
 * @throws LeaderRetrievalException If the actor gateway could not be retrieved or the timeout has been exceeded
 */
public static ActorGateway retrieveLeaderGateway(
		LeaderRetrievalService leaderRetrievalService,
		ActorSystem actorSystem,
		FiniteDuration timeout)
	throws LeaderRetrievalException {
	LeaderGatewayListener listener = new LeaderGatewayListener(actorSystem, timeout);

	try {
		leaderRetrievalService.start(listener);

		Future<ActorGateway> actorGatewayFuture = listener.getActorGatewayFuture();

		return Await.result(actorGatewayFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader gateway.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 

/**
 * Add the given job to be monitored. This means that the service tries to detect leaders for
 * this job and then tries to establish a connection to it.
 *
 * @param jobId identifying the job to monitor
 * @param defaultTargetAddress of the job leader
 * @throws Exception if an error occurs while starting the leader retrieval service
 */
public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception {
	Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running.");

	LOG.info("Add job {} for job leader monitoring.", jobId);

	final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
		jobId,
		defaultTargetAddress);

	JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId);

	final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener));

	if (oldEntry != null) {
		oldEntry.f0.stop();
		oldEntry.f1.stop();
	}

	leaderRetrievalService.start(jobManagerLeaderListener);
}
 

private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 

private JobLeaderIdListener(
		JobID jobId,
		JobLeaderIdActions listenerJobLeaderIdActions,
		LeaderRetrievalService leaderRetrievalService) throws Exception {
	this.jobId = Preconditions.checkNotNull(jobId);
	this.listenerJobLeaderIdActions = Preconditions.checkNotNull(listenerJobLeaderIdActions);
	this.leaderRetrievalService = Preconditions.checkNotNull(leaderRetrievalService);

	leaderIdFuture = new CompletableFuture<>();

	activateTimeout();

	// start the leader service we're listening to
	leaderRetrievalService.start(this);
}
 

private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 

private JobLeaderIdListener(
		JobID jobId,
		JobLeaderIdActions listenerJobLeaderIdActions,
		LeaderRetrievalService leaderRetrievalService) throws Exception {
	this.jobId = Preconditions.checkNotNull(jobId);
	this.listenerJobLeaderIdActions = Preconditions.checkNotNull(listenerJobLeaderIdActions);
	this.leaderRetrievalService = Preconditions.checkNotNull(leaderRetrievalService);

	leaderIdFuture = new CompletableFuture<>();

	activateTimeout();

	// start the leader service we're listening to
	leaderRetrievalService.start(this);
}
 

@Test
public void testShutdown() throws Exception {
	final UUID uuid = UUID.randomUUID();
	final SingleLeaderElectionService service = new SingleLeaderElectionService(executor, uuid);

	// create a leader contender and let it grab leadership
	final LeaderContender contender = mockContender(service);
	service.start(contender);
	verify(contender, times(1)).grantLeadership(uuid);

	// some leader listeners
	final LeaderRetrievalListener listener1 = mock(LeaderRetrievalListener.class);
	final LeaderRetrievalListener listener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService listenerService1 = service.createLeaderRetrievalService();
	LeaderRetrievalService listenerService2 = service.createLeaderRetrievalService();

	listenerService1.start(listener1);
	listenerService2.start(listener2);

	// one listener stops
	listenerService1.stop();

	// shut down the service
	service.shutdown();

	// the leader contender and running listener should get error notifications
	verify(contender, times(1)).handleError(any(Exception.class));
	verify(listener2, times(1)).handleError(any(Exception.class));

	// the stopped listener gets no notification
	verify(listener1, times(0)).handleError(any(Exception.class));

	// should not be possible to start again after shutdown
	try {
		service.start(contender);
		fail("should fail with an exception");
	} catch (IllegalStateException e) {
		// expected
	}

	// no additional leadership grant
	verify(contender, times(1)).grantLeadership(any(UUID.class));
}
 

private void runLeaderRetrievalTest(LeaderElectionService leaderElectionService, LeaderRetrievalService leaderRetrievalService) throws Exception {
	LeaderRetrievalUtils.LeaderConnectionInfoListener leaderRetrievalListener = new LeaderRetrievalUtils.LeaderConnectionInfoListener();
	TestingLeaderContender leaderContender = new TestingLeaderContender();

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	final UUID leaderId = leaderContender.getLeaderSessionFuture().get();

	leaderElectionService.confirmLeadership(leaderId, ADDRESS);

	final LeaderConnectionInfo leaderConnectionInfo = leaderRetrievalListener.getLeaderConnectionInfoFuture().get();

	assertThat(leaderConnectionInfo.getAddress(), is(ADDRESS));
	assertThat(leaderConnectionInfo.getLeaderSessionId(), is(leaderId));
}