Java Code Examples for org.apache.flink.runtime.leaderelection.LeaderElectionService

The following examples show how to use org.apache.flink.runtime.leaderelection.LeaderElectionService. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: EmbeddedLeaderServiceTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the {@link EmbeddedLeaderService} can handle a concurrent grant
 * leadership call and a shutdown.
 */
@Test
public void testConcurrentGrantLeadershipAndShutdown() throws Exception {
	final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor());

	try {
		final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService();

		final TestingLeaderContender contender = new TestingLeaderContender();

		leaderElectionService.start(contender);
		leaderElectionService.stop();

		try {
			// check that no exception occurred
			contender.getLeaderSessionFuture().get(10L, TimeUnit.MILLISECONDS);
		} catch (TimeoutException ignored) {
			// we haven't participated in the leader election
		}

		// the election service should still be running
		Assert.assertThat(embeddedLeaderService.isShutdown(), is(false));
	} finally {
		embeddedLeaderService.shutdown();
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: DispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 
Example 3
Source Project: Flink-CEPplus   Source File: MiniDispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public MiniDispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<? extends RestfulGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {
	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);
}
 
Example 4
Source Project: flink   Source File: StandaloneHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader election services return a fixed address and leader session
 * id.
 */
@Test
public void testLeaderElection() throws Exception {
	JobID jobId = new JobID();
	LeaderContender jmLeaderContender = mock(LeaderContender.class);
	LeaderContender rmLeaderContender = mock(LeaderContender.class);

	LeaderElectionService jmLeaderElectionService = standaloneHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderElectionService rmLeaderElectionService = standaloneHaServices.getResourceManagerLeaderElectionService();

	jmLeaderElectionService.start(jmLeaderContender);
	rmLeaderElectionService.start(rmLeaderContender);

	verify(jmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example 5
Source Project: Flink-CEPplus   Source File: EmbeddedLeaderServiceTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the {@link EmbeddedLeaderService} can handle a concurrent grant
 * leadership call and a shutdown.
 */
@Test
public void testConcurrentGrantLeadershipAndShutdown() throws Exception {
	final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor());

	try {
		final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService();

		final TestingLeaderContender contender = new TestingLeaderContender();

		leaderElectionService.start(contender);
		leaderElectionService.stop();

		try {
			// check that no exception occurred
			contender.getLeaderSessionFuture().get(10L, TimeUnit.MILLISECONDS);
		} catch (TimeoutException ignored) {
			// we haven't participated in the leader election
		}

		// the election service should still be running
		Assert.assertThat(embeddedLeaderService.isShutdown(), is(false));
	} finally {
		embeddedLeaderService.shutdown();
	}
}
 
Example 6
Source Project: Flink-CEPplus   Source File: EmbeddedHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that exactly one ResourceManager is elected as the leader.
 */
@Test
public void testResourceManagerLeaderElection() throws Exception {
	LeaderContender leaderContender1 = mock(LeaderContender.class);
	LeaderContender leaderContender2 = mock(LeaderContender.class);

	LeaderElectionService leaderElectionService1 = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderElectionService leaderElectionService2 = embeddedHaServices.getResourceManagerLeaderElectionService();

	leaderElectionService1.start(leaderContender1);
	leaderElectionService2.start(leaderContender2);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor1 = ArgumentCaptor.forClass(UUID.class);
	ArgumentCaptor<UUID> leaderIdArgumentCaptor2 = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender1, atLeast(0)).grantLeadership(leaderIdArgumentCaptor1.capture());
	verify(leaderContender2, atLeast(0)).grantLeadership(leaderIdArgumentCaptor2.capture());

	assertTrue(leaderIdArgumentCaptor1.getAllValues().isEmpty() ^ leaderIdArgumentCaptor2.getAllValues().isEmpty());
}
 
Example 7
Source Project: Flink-CEPplus   Source File: EmbeddedHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the JobManager leader retrieval for a given job.
 */
@Test
public void testJobManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	JobID jobId = new JobID();
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getJobManagerLeaderRetriever(jobId);

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 
Example 8
Source Project: Flink-CEPplus   Source File: EmbeddedHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the ResourceManager leader retrieval for a given job.
 */
@Test
public void testResourceManagerLeaderRetrieval() throws Exception {
	final String address = "foobar";
	LeaderRetrievalListener leaderRetrievalListener = mock(LeaderRetrievalListener.class);
	LeaderContender leaderContender = mock(LeaderContender.class);
	when(leaderContender.getAddress()).thenReturn(address);

	LeaderElectionService leaderElectionService = embeddedHaServices.getResourceManagerLeaderElectionService();
	LeaderRetrievalService leaderRetrievalService = embeddedHaServices.getResourceManagerLeaderRetriever();

	leaderRetrievalService.start(leaderRetrievalListener);
	leaderElectionService.start(leaderContender);

	ArgumentCaptor<UUID> leaderIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);
	verify(leaderContender).grantLeadership(leaderIdArgumentCaptor.capture());

	final UUID leaderId = leaderIdArgumentCaptor.getValue();

	leaderElectionService.confirmLeaderSessionID(leaderId);

	verify(leaderRetrievalListener).notifyLeaderAddress(eq(address), eq(leaderId));
}
 
Example 9
Source Project: Flink-CEPplus   Source File: StandaloneHaServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the standalone leader election services return a fixed address and leader session
 * id.
 */
@Test
public void testLeaderElection() throws Exception {
	JobID jobId = new JobID();
	LeaderContender jmLeaderContender = mock(LeaderContender.class);
	LeaderContender rmLeaderContender = mock(LeaderContender.class);

	LeaderElectionService jmLeaderElectionService = standaloneHaServices.getJobManagerLeaderElectionService(jobId);
	LeaderElectionService rmLeaderElectionService = standaloneHaServices.getResourceManagerLeaderElectionService();

	jmLeaderElectionService.start(jmLeaderContender);
	rmLeaderElectionService.start(rmLeaderContender);

	verify(jmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
	verify(rmLeaderContender).grantLeadership(eq(HighAvailabilityServices.DEFAULT_LEADER_ID));
}
 
Example 10
Source Project: flink   Source File: YarnIntraNonHaMasterServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testClosingReportsToLeader() throws Exception {
	final Configuration flinkConfig = new Configuration();

	try (YarnHighAvailabilityServices services = new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig)) {
		final LeaderElectionService elector = services.getResourceManagerLeaderElectionService();
		final LeaderRetrievalService retrieval = services.getResourceManagerLeaderRetriever();
		final LeaderContender contender = mockContender(elector);
		final LeaderRetrievalListener listener = mock(LeaderRetrievalListener.class);

		elector.start(contender);
		retrieval.start(listener);

		// wait until the contender has become the leader
		verify(listener, timeout(1000L).times(1)).notifyLeaderAddress(anyString(), any(UUID.class));

		// now we can close the election service
		services.close();

		verify(contender, timeout(1000L).times(1)).handleError(any(Exception.class));
	}
}
 
Example 11
Source Project: flink   Source File: DispatcherRestEndpoint.java    License: Apache License 2.0 6 votes vote down vote up
public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 
Example 12
Source Project: flink   Source File: DefaultDispatcherRunnerFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public DispatcherRunner createDispatcherRunner(
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler,
		JobGraphStoreFactory jobGraphStoreFactory,
		Executor ioExecutor,
		RpcService rpcService,
		PartialDispatcherServices partialDispatcherServices) throws Exception {

	final DispatcherLeaderProcessFactory dispatcherLeaderProcessFactory = dispatcherLeaderProcessFactoryFactory.createFactory(
		jobGraphStoreFactory,
		ioExecutor,
		rpcService,
		partialDispatcherServices,
		fatalErrorHandler);

	return DefaultDispatcherRunner.create(
		leaderElectionService,
		fatalErrorHandler,
		dispatcherLeaderProcessFactory);
}
 
Example 13
@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	enter();
	try {
		return resourceManagerLeaderElectionService;
	}
	finally {
		exit();
	}
}
 
Example 14
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 15
@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	enter();
	try {
		throw new UnsupportedOperationException("Not supported on the TaskManager side");
	}
	finally {
		exit();
	}
}
 
Example 16
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 
Example 17
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	enter();
	try {
		throw new UnsupportedOperationException("needs refactoring to accept default address");
	}
	finally {
		exit();
	}
}
 
Example 18
@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	enter();
	try {
		throw new UnsupportedOperationException();
	}
	finally {
		exit();
	}
}
 
Example 19
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getWebMonitorLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: WebMonitorEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
public WebMonitorEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<? extends T> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		FatalErrorHandler fatalErrorHandler) throws IOException {
	super(endpointConfiguration);
	this.leaderRetriever = Preconditions.checkNotNull(leaderRetriever);
	this.clusterConfiguration = Preconditions.checkNotNull(clusterConfiguration);
	this.restConfiguration = Preconditions.checkNotNull(restConfiguration);
	this.resourceManagerRetriever = Preconditions.checkNotNull(resourceManagerRetriever);
	this.transientBlobService = Preconditions.checkNotNull(transientBlobService);
	this.executor = Preconditions.checkNotNull(executor);

	this.executionGraphCache = new ExecutionGraphCache(
		restConfiguration.getTimeout(),
		Time.milliseconds(restConfiguration.getRefreshInterval()));

	this.checkpointStatsCache = new CheckpointStatsCache(
		restConfiguration.getMaxCheckpointStatisticCacheEntries());

	this.metricFetcher = metricFetcher;

	this.leaderElectionService = Preconditions.checkNotNull(leaderElectionService);
	this.fatalErrorHandler = Preconditions.checkNotNull(fatalErrorHandler);
}
 
Example 21
Source Project: flink   Source File: DispatcherRestEndpoint.java    License: Apache License 2.0 5 votes vote down vote up
public DispatcherRestEndpoint(
		RestServerEndpointConfiguration endpointConfiguration,
		GatewayRetriever<DispatcherGateway> leaderRetriever,
		Configuration clusterConfiguration,
		RestHandlerConfiguration restConfiguration,
		GatewayRetriever<ResourceManagerGateway> resourceManagerRetriever,
		TransientBlobService transientBlobService,
		ScheduledExecutorService executor,
		MetricFetcher metricFetcher,
		LeaderElectionService leaderElectionService,
		ExecutionGraphCache executionGraphCache,
		FatalErrorHandler fatalErrorHandler) throws IOException {

	super(
		endpointConfiguration,
		leaderRetriever,
		clusterConfiguration,
		restConfiguration,
		resourceManagerRetriever,
		transientBlobService,
		executor,
		metricFetcher,
		leaderElectionService,
		executionGraphCache,
		fatalErrorHandler);

	webSubmissionExtension = WebMonitorExtension.empty();
}
 
Example 22
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 
Example 23
Source Project: Flink-CEPplus   Source File: EmbeddedHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	checkNotNull(jobID);

	synchronized (lock) {
		checkNotShutdown();
		EmbeddedLeaderService service = getOrCreateJobManagerService(jobID);
		return service.createLeaderElectionService();
	}
}
 
Example 24
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getDispatcherLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 
Example 25
Source Project: Flink-CEPplus   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 
Example 26
Source Project: flink   Source File: DispatcherRunnerFactory.java    License: Apache License 2.0 5 votes vote down vote up
DispatcherRunner createDispatcherRunner(
LeaderElectionService leaderElectionService,
FatalErrorHandler fatalErrorHandler,
JobGraphStoreFactory jobGraphStoreFactory,
Executor ioExecutor,
RpcService rpcService,
PartialDispatcherServices partialDispatcherServices) throws Exception;
 
Example 27
private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpMetricRegistry.INSTANCE,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	return resourceManager;
}
 
Example 28
Source Project: flink   Source File: StandaloneHaServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getResourceManagerLeaderElectionService() {
	synchronized (lock) {
		checkNotShutdown();

		return new StandaloneLeaderElectionService();
	}
}
 
Example 29
Source Project: flink   Source File: TestingHighAvailabilityServices.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeaderElectionService getJobManagerLeaderElectionService(JobID jobID) {
	LeaderElectionService service = jobManagerLeaderElectionServices.computeIfAbsent(jobID, jobMasterLeaderElectionServiceFunction);

	if (service != null) {
		return service;
	} else {
		throw new IllegalStateException("JobMasterLeaderElectionService has not been set");
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: EmbeddedLeaderServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the {@link EmbeddedLeaderService} can handle a concurrent revoke
 * leadership call and a shutdown.
 */
@Test
public void testConcurrentRevokeLeadershipAndShutdown() throws Exception {
	final EmbeddedLeaderService embeddedLeaderService = new EmbeddedLeaderService(TestingUtils.defaultExecutor());

	try {
		final LeaderElectionService leaderElectionService = embeddedLeaderService.createLeaderElectionService();

		final TestingLeaderContender contender = new TestingLeaderContender();

		leaderElectionService.start(contender);

		// wait for the leadership
		contender.getLeaderSessionFuture().get();

		final CompletableFuture<Void> revokeLeadershipFuture = embeddedLeaderService.revokeLeadership();
		leaderElectionService.stop();

		try {
			// check that no exception occurred
			revokeLeadershipFuture.get(10L, TimeUnit.MILLISECONDS);
		} catch (TimeoutException ignored) {
			// the leader election service has been stopped before revoking could be executed
		}

		// the election service should still be running
		Assert.assertThat(embeddedLeaderService.isShutdown(), is(false));
	} finally {
		embeddedLeaderService.shutdown();
	}
}