Java Code Examples for org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService

The following examples show how to use org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: JobLeaderService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Add the given job to be monitored. This means that the service tries to detect leaders for
 * this job and then tries to establish a connection to it.
 *
 * @param jobId identifying the job to monitor
 * @param defaultTargetAddress of the job leader
 * @throws Exception if an error occurs while starting the leader retrieval service
 */
public void addJob(final JobID jobId, final String defaultTargetAddress) throws Exception {
	Preconditions.checkState(JobLeaderService.State.STARTED == state, "The service is currently not running.");

	LOG.info("Add job {} for job leader monitoring.", jobId);

	final LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(
		jobId,
		defaultTargetAddress);

	JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = new JobManagerLeaderListener(jobId);

	final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> oldEntry = jobLeaderServices.put(jobId, Tuple2.of(leaderRetrievalService, jobManagerLeaderListener));

	if (oldEntry != null) {
		oldEntry.f0.stop();
		oldEntry.f1.stop();
	}

	leaderRetrievalService.start(jobManagerLeaderListener);
}
 
Example 2
Source Project: flink   Source File: StandaloneHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader retrieval services return the specified address and the
 * fixed leader session id.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener rmListener = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2);
	LeaderRetrievalService rmLeaderRetrievalService = standaloneHaServices.getResourceManagerLeaderRetriever();

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);
	rmLeaderRetrievalService.start(rmListener);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmListener).notifyLeaderAddress(eq(resourceManagerAddress), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example 3
@Override
protected DispatcherResourceManagerComponent<Dispatcher> createDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	return new SessionDispatcherResourceManagerComponent(
		dispatcher,
		resourceManager,
		dispatcherLeaderRetrievalService,
		resourceManagerRetrievalService,
		webMonitorEndpoint,
		jobManagerMetricGroup);
}
 
Example 4
Source Project: flink   Source File: JobLeaderService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Stop the job leader services. This implies stopping all leader retrieval services for the
 * different jobs and their leader retrieval listeners.
 *
 * @throws Exception if an error occurs while stopping the service
 */
public void stop() throws Exception {
	LOG.info("Stop job leader service.");

	if (JobLeaderService.State.STARTED == state) {

		for (Tuple2<LeaderRetrievalService, JobLeaderService.JobManagerLeaderListener> leaderRetrievalServiceEntry: jobLeaderServices.values()) {
			LeaderRetrievalService leaderRetrievalService = leaderRetrievalServiceEntry.f0;
			JobLeaderService.JobManagerLeaderListener jobManagerLeaderListener = leaderRetrievalServiceEntry.f1;

			jobManagerLeaderListener.stop();
			leaderRetrievalService.stop();
		}

		jobLeaderServices.clear();
	}

	state = JobLeaderService.State.STOPPED;
}
 
Example 5
Source Project: Flink-CEPplus   Source File: LeaderRetrievalUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the current leader gateway using the given {@link LeaderRetrievalService}. If the
 * current leader could not be retrieved after the given timeout, then a
 * {@link LeaderRetrievalException} is thrown.
 *
 * @param leaderRetrievalService {@link LeaderRetrievalService} which is used for the leader retrieval
 * @param actorSystem ActorSystem which is used for the {@link LeaderRetrievalListener} implementation
 * @param timeout Timeout value for the retrieval call
 * @return The current leader gateway
 * @throws LeaderRetrievalException If the actor gateway could not be retrieved or the timeout has been exceeded
 */
public static ActorGateway retrieveLeaderGateway(
		LeaderRetrievalService leaderRetrievalService,
		ActorSystem actorSystem,
		FiniteDuration timeout)
	throws LeaderRetrievalException {
	LeaderGatewayListener listener = new LeaderGatewayListener(actorSystem, timeout);

	try {
		leaderRetrievalService.start(listener);

		Future<ActorGateway> actorGatewayFuture = listener.getActorGatewayFuture();

		return Await.result(actorGatewayFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader gateway.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 
Example 6
Source Project: Flink-CEPplus   Source File: LeaderRetrievalUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the leader akka url and the current leader session ID. The values are stored in a
 * {@link LeaderConnectionInfo} instance.
 *
 * @param leaderRetrievalService Leader retrieval service to retrieve the leader connection
 *                               information
 * @param timeout Timeout when to give up looking for the leader
 * @return LeaderConnectionInfo containing the leader's akka URL and the current leader session
 * ID
 * @throws LeaderRetrievalException
 */
public static LeaderConnectionInfo retrieveLeaderConnectionInfo(
		LeaderRetrievalService leaderRetrievalService,
		FiniteDuration timeout
) throws LeaderRetrievalException {
	LeaderConnectionInfoListener listener = new LeaderConnectionInfoListener();

	try {
		leaderRetrievalService.start(listener);

		Future<LeaderConnectionInfo> connectionInfoFuture = listener.getLeaderConnectionInfoFuture();

		return Await.result(connectionInfoFuture, timeout);
	} catch (Exception e) {
		throw new LeaderRetrievalException("Could not retrieve the leader address and leader " +
				"session ID.", e);
	} finally {
		try {
			leaderRetrievalService.stop();
		} catch (Exception fe) {
			LOG.warn("Could not stop the leader retrieval service.", fe);
		}
	}
}
 
Example 7
Source Project: flink   Source File: StandaloneHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader retrieval services return the given address and the
 * fixed leader session id.
 */
@Test
public void testJobMasterLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	final String jobManagerAddress1 = "foobar";
	final String jobManagerAddress2 = "barfoo";
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2);

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example 8
Source Project: flink   Source File: EmbeddedHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	JobID jobId = new JobID();
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 
Example 9
Source Project: Flink-CEPplus   Source File: EmbeddedHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the ResourceManager leader retrieval for a given job.
 */
@Test
public void testResourceManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever();

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 
Example 10
Source Project: Flink-CEPplus   Source File: StandaloneHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader retrieval services return the given address and the
 * fixed leader session id.
 */
@Test
public void testJobMasterLeaderRetrieval() throws Exception {
	JobID jobId1 = new JobID();
	JobID jobId2 = new JobID();
	final String jobManagerAddress1 = "foobar";
	final String jobManagerAddress2 = "barfoo";
	LeaderRetrievalListener jmListener1 = mock(LeaderRetrievalListener.class);
	LeaderRetrievalListener jmListener2 = mock(LeaderRetrievalListener.class);

	LeaderRetrievalService jmLeaderRetrievalService1 = standaloneHaServices.getJobManagerLeaderRetriever(jobId1, jobManagerAddress1);
	LeaderRetrievalService jmLeaderRetrievalService2 = standaloneHaServices.getJobManagerLeaderRetriever(jobId2, jobManagerAddress2);

	jmLeaderRetrievalService1.start(jmListener1);
	jmLeaderRetrievalService2.start(jmListener2);

	verify(jmListener1).notifyLeaderAddress(eq(jobManagerAddress1), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(jmListener2).notifyLeaderAddress(eq(jobManagerAddress2), eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example 11
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 12
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 13
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(resourceManagerRpcUrl, DEFAULT_LEADER_ID);
	}
	finally {
		exit();
	}
}
 
Example 14
@Override
public LeaderRetrievalService getDispatcherLeaderRetriever() {
	enter();

	try {
		return new StandaloneLeaderRetrievalService(dispatcherRpcUrl, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 
Example 15
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 16
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	} finally {
		exit();
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: RestClusterClient.java    License: Apache License 2.0 5 votes vote down vote up
public RestClusterClient(
		Configuration config,
		T clusterId,
		LeaderRetrievalService webMonitorRetrievalService) throws Exception {
	this(
		config,
		null,
		clusterId,
		new ExponentialWaitStrategy(10L, 2000L),
		webMonitorRetrievalService);
}
 
Example 18
Source Project: flink   Source File: ZooKeeperLeaderRetrievalTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the LeaderRetrievalUtils.findConnectingAddress stops trying to find the
 * connecting address if no leader address has been specified. The call should return
 * then InetAddress.getLocalHost().
 */
@Test
public void testTimeoutOfFindConnectingAddress() throws Exception {
	Duration timeout = Duration.ofSeconds(1L);

	LeaderRetrievalService leaderRetrievalService = highAvailabilityServices.getJobManagerLeaderRetriever(HighAvailabilityServices.DEFAULT_JOB_ID);
	InetAddress result = LeaderRetrievalUtils.findConnectingAddress(leaderRetrievalService, timeout);

	assertEquals(InetAddress.getLocalHost(), result);
}
 
Example 19
SessionDispatcherResourceManagerComponent(
		Dispatcher dispatcher,
		ResourceManager<?> resourceManager,
		LeaderRetrievalService dispatcherLeaderRetrievalService,
		LeaderRetrievalService resourceManagerRetrievalService,
		WebMonitorEndpoint<?> webMonitorEndpoint,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(dispatcher, resourceManager, dispatcherLeaderRetrievalService, resourceManagerRetrievalService, webMonitorEndpoint, jobManagerMetricGroup);
}
 
Example 20
Source Project: flink   Source File: DefaultJobLeaderService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void reconnect(final JobID jobId) {
	Preconditions.checkNotNull(jobId, "JobID must not be null.");

	final Tuple2<LeaderRetrievalService, JobManagerLeaderListener> jobLeaderService = jobLeaderServices.get(jobId);

	if (jobLeaderService != null) {
		jobLeaderService.f1.reconnect();
	} else {
		LOG.info("Cannot reconnect to job {} because it is not registered.", jobId);
	}
}
 
Example 21
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 22
Source Project: flink   Source File: ZooKeeperHaServicesTest.java    License: Apache License 2.0 5 votes vote down vote up
private void runCleanupTest(
		Configuration configuration,
		TestingBlobStoreService blobStoreService,
		ThrowingConsumer<ZooKeeperHaServices, Exception> zooKeeperHaServicesConsumer) throws Exception {
	try (ZooKeeperHaServices zooKeeperHaServices = new ZooKeeperHaServices(
		ZooKeeperUtils.startCuratorFramework(configuration),
		Executors.directExecutor(),
		configuration,
		blobStoreService)) {

		// create some Zk services to trigger the generation of paths
		final LeaderRetrievalService resourceManagerLeaderRetriever = zooKeeperHaServices.getResourceManagerLeaderRetriever();
		final LeaderElectionService resourceManagerLeaderElectionService = zooKeeperHaServices.getResourceManagerLeaderElectionService();
		final RunningJobsRegistry runningJobsRegistry = zooKeeperHaServices.getRunningJobsRegistry();

		final TestingListener listener = new TestingListener();
		resourceManagerLeaderRetriever.start(listener);
		resourceManagerLeaderElectionService.start(new TestingContender("foobar", resourceManagerLeaderElectionService));
		final JobID jobId = new JobID();
		runningJobsRegistry.setJobRunning(jobId);

		listener.waitForNewLeader(2000L);

		resourceManagerLeaderRetriever.stop();
		resourceManagerLeaderElectionService.stop();
		runningJobsRegistry.clearJob(jobId);

		zooKeeperHaServicesConsumer.accept(zooKeeperHaServices);
	}
}
 
Example 23
Source Project: flink   Source File: YarnIntraNonHaMasterServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 24
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 25
Source Project: flink   Source File: ResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testHeartbeatTimeoutWithJobMaster() throws Exception {
	final CompletableFuture<ResourceID> heartbeatRequestFuture = new CompletableFuture<>();
	final CompletableFuture<ResourceManagerId> disconnectFuture = new CompletableFuture<>();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
		.setResourceManagerHeartbeatConsumer(heartbeatRequestFuture::complete)
		.setDisconnectResourceManagerConsumer(disconnectFuture::complete)
		.build();
	rpcService.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);
	final JobID jobId = new JobID();
	final ResourceID jobMasterResourceId = ResourceID.generate();
	final LeaderRetrievalService jobMasterLeaderRetrievalService = new SettableLeaderRetrievalService(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());

	highAvailabilityServices.setJobMasterLeaderRetrieverFunction(requestedJobId -> {
		assertThat(requestedJobId, is(equalTo(jobId)));
		return jobMasterLeaderRetrievalService;
	});

	runHeartbeatTimeoutTest(
		resourceManagerGateway -> {
			final CompletableFuture<RegistrationResponse> registrationFuture = resourceManagerGateway.registerJobManager(
				jobMasterGateway.getFencingToken(),
				jobMasterResourceId,
				jobMasterGateway.getAddress(),
				jobId,
				TIMEOUT);

			assertThat(registrationFuture.get(), instanceOf(RegistrationResponse.Success.class));
		},
		resourceManagerResourceId -> {
			// might have been completed or not depending whether the timeout was triggered first
			final ResourceID optionalHeartbeatRequestOrigin = heartbeatRequestFuture.getNow(null);
			assertThat(optionalHeartbeatRequestOrigin, anyOf(is(resourceManagerResourceId), is(nullValue())));
			assertThat(disconnectFuture.get(), is(equalTo(resourceManagerId)));
		});
}
 
Example 26
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID, String defaultJobManagerAddress) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(defaultJobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 27
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getResourceManagerLeaderRetriever() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(resourceManagerAddress, DEFAULT_LEADER_ID);
	}

}
 
Example 28
Source Project: flink   Source File: YarnIntraNonHaMasterServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getJobManagerLeaderRetriever(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderRetrievalService(jobManagerAddress, DEFAULT_LEADER_ID);
	}
}
 
Example 30
Source Project: flink   Source File: StandaloneClientHAServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderRetrievalService getClusterRestEndpointLeaderRetriever() {
	synchronized (lock) {
		checkState(running, "ClientHaService has already been closed.");
		return new StandaloneLeaderRetrievalService(webMonitorAddress, DEFAULT_LEADER_ID);
	}
}