org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService Java Examples

@Override
protected DispatcherResourceManagerComponent<Dispatcher> createDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	return new SessionDispatcherResourceManagerComponent(
		dispatcher,
		resourceManager,
		dispatcherLeaderRetrievalService,
		resourceManagerRetrievalService,
		webMonitorEndpoint,
		jobManagerMetricGroup);
}
 

/**
 * Tests that the standalone leader retrieval services return the given address and the
 * fixed leader session id.
 */
@Test
public void testJobMasterLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	final String jobManagerAddress1 = "foobar";
	final String jobManagerAddress2 = "barfoo";
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2);

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * Tests that the standalone leader retrieval services return the specified address and the
 * fixed leader session id.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
	LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);
	rmLeaderRetrievalService.start(rmListener);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * Retrieves the leader akka url and the current leader session ID. The values are stored in a
 * {@link LeaderConnectionInfo} instance.
 *
 * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection
 *                               information
 * @param timeout Timeout when to give up looking for the leader
 * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session
 * ID
 * @throws LeaderRetrievalException
 */
public static LeaderConnectionInfo retrieveLeaderConnectionInfo(
		LeaderRetrievalService leaderRetrievalService,
		FiniteDuration timeout
) throws LeaderRetrievalException {
	LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener();

	try {
		leaderRetrievalService.start(listener);

		Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture();

		return Await.result(connectionInfoFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader address and leader " +
				"session ID.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 

/**
 * Tests the ResourceManager leader retrieval for a given job.
 */
@Test
public void testResourceManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever();

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Retrieves the current leader gateway using the given {@link LeaderRetrievalService}. If the
 * current leader could not be retrieved after the given timeout, then a
 * {@link LeaderRetrievalException} is thrown.
 *
 * @param leaderRetrievalService {@link LeaderRetrievalService} which is used for the leader retrieval
 * @param actorSystem ActorSystem which is used for the {@link LeaderRetrievalListener} implementation
 * @param timeout Timeout value for the retrieval call
 * @return The current leader gateway
 * @throws LeaderRetrievalException If the actor gateway could not be retrieved or the timeout has been exceeded
 */
public static ActorGateway retrieveLeaderGateway(
		LeaderRetrievalService leaderRetrievalService,
		ActorSystem actorSystem,
		FiniteDuration timeout)
	throws LeaderRetrievalException {
	LeaderGatewayListener listener = new LeaderGatewayListener(actorSystem, timeout);

	try {
		leaderRetrievalService.start(listener);

		Future<ActorGateway> actorGatewayFuture = listener.getActorGatewayFuture();

		return Await.result(actorGatewayFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader gateway.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 

/**
 * Add the given job to be monitored. This means that the service tries to detect leaders for
 * this job and then tries to establish a connection to it.
 *
 * @param jobId identifying the job to monitor
 * @param defaultTargetAddress of the job leader
 * @throws Exception if an error occurs while starting the leader retrieval service
 */
public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception {
	Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running.");

	LOG.info("Add job {} for job leader monitoring.", jobId);

	final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
		jobId,
		defaultTargetAddress);

	JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId);

	final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener));

	if (oldEntry != null) {
		oldEntry.f0.stop();
		oldEntry.f1.stop();
	}

	leaderRetrievalService.start(jobManagerLeaderListener);
}
 

/**
 * Stop the job leader services. This implies stopping all leader retrieval services for the
 * different jobs and their leader retrieval listeners.
 *
 * @throws Exception if an error occurs while stopping the service
 */
public void stop() throws Exception {
	LOG.info("Stop job leader service.");

	if (JobLeaderService.State.STARTED == state) {

		for (Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener> leaderRetrievalServiceEntry: jobLeaderServices.values()) {
			LeaderRetrievalService leaderRetrievalService = leaderRetrievalServiceEntry.f0;
			JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = leaderRetrievalServiceEntry.f1;

			jobManagerLeaderListener.stop();
			leaderRetrievalService.stop();
		}

		jobLeaderServices.clear();
	}

	state = JobLeaderService.State.STOPPED;
}
 

/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	JobID jobId = new JobID();
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 

/**
 * Tests that the standalone leader retrieval services return the given address and the
 * fixed leader session id.
 */
@Test
public void testJobMasterLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	final String jobManagerAddress1 = "foobar";
	final String jobManagerAddress2 = "barfoo";
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2);

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 

/**
 * This retriever should no longer be used on the cluster side. The web monitor retriever
 * is only required on the client-side and we have a dedicated high-availability services
 * for the client, named {@link ClientHighAvailabilityServices}. See also FLINK-13750.
 *
 * @return the leader retriever for web monitor
 * @deprecated just use {@link #getClusterRestEndpointLeaderRetriever()} instead of this method.
 */
@Deprecated
default LeaderRetrievalService getWebMonitorLeaderRetriever() {
	throw new UnsupportedOperationException(
		"getWebMonitorLeaderRetriever should no longer be used. Instead use " +
			"#getClusterRestEndpointLeaderRetriever to instantiate the cluster " +
			"rest endpoint leader retriever. If you called this method, then " +
			"make sure that #getClusterRestEndpointLeaderRetriever has been " +
			"implemented by your HighAvailabilityServices implementation.");
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	LeaderRetrievalService service = jobMasterLeaderRetrievers.computeIfAbsent(jobID, jobMasterLeaderRetrieverFunction);
	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("JobMasterLeaderRetriever has not been set");
	}
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	LeaderRetrievalService service = this.dispatcherLeaderRetriever;
	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("ResourceManagerLeaderRetriever has not been set");
	}
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID);
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

/**
 * Tests that the LeaderRetrievalUtils.findConnectingAddress stops trying to find the
 * connecting address if no leader address has been specified. The call should return
 * then InetAddress.getLocalHost().
 */
@Test
public void testTimeoutOfFindConnectingAddress() throws Exception {
	Duration timeout = Duration.ofSeconds(1L);

	LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
	InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);

	assertEquals(InetAddress.getLocalHost(), result);
}
 

@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	enter();

	try {
		return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 

@Test
public void testHeartbeatTimeoutWithJobMaster() throws Exception {
	final CompletableFuture<ResourceID> heartbeatRequestFuture = new CompletableFuture<>();
	final CompletableFuture<ResourceManagerId> disconnectFuture = new CompletableFuture<>();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
		.setResourceManagerHeartbeatConsumer(heartbeatRequestFuture::complete)
		.setDisconnectResourceManagerConsumer(disconnectFuture::complete)
		.build();
	rpcService.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
	final JobID jobId = new JobID();
	final ResourceID jobMasterResourceId = ResourceID.generate();
	final LeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());

	highAvailabilityServices.setJobMasterLeaderRetrieverFunction(requestedJobId -> {
		assertThat(requestedJobId, is(equalTo(jobId)));
		return jobMasterLeaderRetrievalService;
	});

	runHeartbeatTimeoutTest(
		resourceManagerGateway -> {
			final CompletableFuture<RegistrationResponse> registrationFuture = resourceManagerGateway.registerJobManager(
				jobMasterGateway.getFencingToken(),
				jobMasterResourceId,
				jobMasterGateway.getAddress(),
				jobId,
				TIMEOUT);

			assertThat(registrationFuture.get(), instanceOf(RegistrationResponse.Success.class));
		},
		resourceManagerResourceId -> {
			// might have been completed or not depending whether the timeout was triggered first
			final ResourceID optionalHeartbeatRequestOrigin = heartbeatRequestFuture.getNow(null);
			assertThat(optionalHeartbeatRequestOrigin, anyOf(is(resourceManagerResourceId), is(nullValue())));
			assertThat(disconnectFuture.get(), is(equalTo(resourceManagerId)));
		});
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 

@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	synchronized (lock) {
		checkState(running, "ClientHaService has already been closed.");
		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 

@Override
public LeaderRetrievalService getWebMonitorLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}
 

public LeaderRetrievalService createLeaderRetrievalService() {
	final SettableLeaderRetrievalService settableLeaderRetrievalService = new SettableLeaderRetrievalService(
		getLeaderAddress(currentLeaderIndex),
		currentLeaderId);

	leaderRetrievalServices.add(settableLeaderRetrievalService);

	return settableLeaderRetrievalService;
}
 

@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	LeaderRetrievalService service = this.resourceManagerLeaderRetriever;
	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("ResourceManagerLeaderRetriever has not been set");
	}
}
 

/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId = new JobID();

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	runLeaderRetrievalTest(leaderElectionService, leaderRetrievalService);
}
 

public RestClusterClient(
		Configuration config,
		T clusterId,
		LeaderRetrievalService webMonitorRetrievalService) throws Exception {
	this(
		config,
		null,
		clusterId,
		new ExponentialWaitStrategy(10L, 2000L),
		webMonitorRetrievalService);
}
 

@VisibleForTesting
RestClusterClient(
		Configuration configuration,
		@Nullable RestClient restClient,
		T clusterId,
		WaitStrategy waitStrategy,
		@Nullable LeaderRetrievalService webMonitorRetrievalService) throws Exception {
	super(configuration);
	this.restClusterClientConfiguration = RestClusterClientConfiguration.fromConfiguration(configuration);

	if (restClient != null) {
		this.restClient = restClient;
	} else {
		this.restClient = new RestClient(restClusterClientConfiguration.getRestClientConfiguration(), executorService);
	}

	this.waitStrategy = Preconditions.checkNotNull(waitStrategy);
	this.clusterId = Preconditions.checkNotNull(clusterId);

	if (webMonitorRetrievalService == null) {
		this.webMonitorRetrievalService = highAvailabilityServices.getWebMonitorLeaderRetriever();
	} else {
		this.webMonitorRetrievalService = webMonitorRetrievalService;
	}
	this.retryExecutorService = Executors.newSingleThreadScheduledExecutor(new ExecutorThreadFactory("Flink-RestClusterClient-Retry"));
	startLeaderRetrievers();
}