Java Code Examples for org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices

The following examples show how to use org.apache.flink.runtime.highavailability.TestingHighAvailabilityServices. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: MesosResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new HeartbeatServices(5L, 5L);
	metricRegistry = mock(MetricRegistryImpl.class);
	slotManager = mock(SlotManager.class);
	slotManagerStarted = new CompletableFuture<>();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			rmActions = invocation.getArgument(2);
			slotManagerStarted.complete(true);
			return null;
		}
	}).when(slotManager).start(any(ResourceManagerId.class), any(Executor.class), any(ResourceActions.class));

	when(slotManager.registerSlotRequest(any(SlotRequest.class))).thenReturn(true);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new TestingHeartbeatServices();
	metricRegistry = NoOpMetricRegistry.INSTANCE;
	slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(new ScheduledExecutorServiceAdapter(new DirectScheduledExecutorService()))
		.setTaskManagerRequestTimeout(Time.seconds(10))
		.setSlotRequestTimeout(Time.seconds(10))
		.setTaskManagerTimeout(Time.minutes(1))
		.build();
	jobLeaderIdService = new JobLeaderIdService(
			highAvailabilityServices,
			rpcService.getScheduledExecutor(),
			Time.minutes(5L));
}
 
Example 3
Source Project: Flink-CEPplus   Source File: DispatcherHATest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a fatal error is reported if the job recovery fails.
 */
@Test
public void testFailingRecoveryIsAFatalError() throws Exception {
	final String exceptionMessage = "Job recovery test failure.";
	final Supplier<Exception> exceptionSupplier = () -> new FlinkException(exceptionMessage);
	final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder()
		.setSubmittedJobGraphStore(new FailingSubmittedJobGraphStore(exceptionSupplier))
		.build();

	final HATestingDispatcher dispatcher = createDispatcher(haServices);
	dispatcher.start();

	final Throwable failure = testingFatalErrorHandler.getErrorFuture().get();

	assertThat(ExceptionUtils.findThrowableWithMessage(failure, exceptionMessage).isPresent(), is(true));

	testingFatalErrorHandler.clearError();
}
 
Example 4
Source Project: Flink-CEPplus   Source File: DispatcherTest.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private TestingDispatcher createDispatcher(HeartbeatServices heartbeatServices, TestingHighAvailabilityServices haServices, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	return new TestingDispatcher(
		rpcService,
		Dispatcher.DISPATCHER_NAME + '_' + name.getMethodName(),
		configuration,
		haServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		heartbeatServices,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		jobManagerRunnerFactory,
		fatalErrorHandler);
}
 
Example 5
Source Project: Flink-CEPplus   Source File: JobMasterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
 
Example 6
Source Project: Flink-CEPplus   Source File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	rpc = new TestingRpcService();
	timerService = new TimerService<>(TestingUtils.defaultExecutor(), timeout.toMilliseconds());

	dummyBlobCacheService = new BlobCacheService(
		new Configuration(),
		new VoidBlobStore(),
		null);

	configuration = new Configuration();
	taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);

	taskManagerLocation = new LocalTaskManagerLocation();
	jobId = new JobID();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices = new TestingHighAvailabilityServices();
	resourceManagerLeaderRetriever = new SettableLeaderRetrievalService();
	jobManagerLeaderRetriever = new SettableLeaderRetrievalService();
	haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever);
	haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever);
}
 
Example 7
Source Project: flink   Source File: MesosResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new HeartbeatServices(5L, 5L);
	metricRegistry = mock(MetricRegistryImpl.class);
	slotManager = mock(SlotManager.class);
	slotManagerStarted = new CompletableFuture<>();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			rmActions = invocation.getArgument(2);
			slotManagerStarted.complete(true);
			return null;
		}
	}).when(slotManager).start(any(ResourceManagerId.class), any(Executor.class), any(ResourceActions.class));

	when(slotManager.registerSlotRequest(any(SlotRequest.class))).thenReturn(true);
}
 
Example 8
Source Project: flink   Source File: DispatcherHATest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a fatal error is reported if the job recovery fails.
 */
@Test
public void testFailingRecoveryIsAFatalError() throws Exception {
	final String exceptionMessage = "Job recovery test failure.";
	final Supplier<Exception> exceptionSupplier = () -> new FlinkException(exceptionMessage);
	final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder()
		.setSubmittedJobGraphStore(new FailingSubmittedJobGraphStore(exceptionSupplier))
		.build();

	final HATestingDispatcher dispatcher = createDispatcher(haServices);
	dispatcher.start();

	final Throwable failure = testingFatalErrorHandler.getErrorFuture().get();

	assertThat(ExceptionUtils.findThrowableWithMessage(failure, exceptionMessage).isPresent(), is(true));

	testingFatalErrorHandler.clearError();
}
 
Example 9
Source Project: flink   Source File: DispatcherTest.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private TestingDispatcher createDispatcher(HeartbeatServices heartbeatServices, TestingHighAvailabilityServices haServices, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	return new TestingDispatcher(
		rpcService,
		Dispatcher.DISPATCHER_NAME + '_' + name.getMethodName(),
		configuration,
		haServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		heartbeatServices,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		jobManagerRunnerFactory,
		fatalErrorHandler);
}
 
Example 10
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
 
Example 11
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	rpc = new TestingRpcService();
	timerService = new TimerService<>(TestingUtils.defaultExecutor(), timeout.toMilliseconds());

	dummyBlobCacheService = new BlobCacheService(
		new Configuration(),
		new VoidBlobStore(),
		null);

	configuration = new Configuration();
	taskManagerConfiguration = TaskManagerConfiguration.fromConfiguration(configuration);

	taskManagerLocation = new LocalTaskManagerLocation();
	jobId = new JobID();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices = new TestingHighAvailabilityServices();
	resourceManagerLeaderRetriever = new SettableLeaderRetrievalService();
	jobManagerLeaderRetriever = new SettableLeaderRetrievalService();
	haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever);
	haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever);

	nettyShuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
}
 
Example 12
@Test
public void createResourceManager_WithLessMemoryThanContainerizedHeapCutoffMin_ShouldSucceed() throws Exception {
	final StandaloneResourceManagerFactory resourceManagerFactory = StandaloneResourceManagerFactory.INSTANCE;

	final TestingRpcService rpcService = new TestingRpcService();
	try {
		final Configuration configuration = new Configuration();
		configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, new MemorySize(128 * 1024 * 1024).toString());
		configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 600);

		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 
Example 13
Source Project: flink   Source File: ActiveResourceManagerFactoryTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test which ensures that the {@link ActiveResourceManagerFactory} sets the correct managed
 * memory when creating a resource manager.
 */
@Test
public void createResourceManager_WithDefaultConfiguration_ShouldSetManagedMemory() throws Exception {
	final Configuration configuration = new Configuration();

	final TestingActiveResourceManagerFactory resourceManagerFactory = new TestingActiveResourceManagerFactory();

	final TestingRpcService rpcService = new TestingRpcService();

	try {
		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 
Example 14
Source Project: flink   Source File: MesosResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
MockResourceManagerRuntimeServices() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	rmLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);
	heartbeatServices = new HeartbeatServices(Integer.MAX_VALUE, Integer.MAX_VALUE);
	slotManager = mock(SlotManager.class);
	slotManagerStarted = new CompletableFuture<>();
	jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	doAnswer(new Answer<Object>() {
		@Override
		public Object answer(InvocationOnMock invocation) throws Throwable {
			rmActions = invocation.getArgument(2);
			slotManagerStarted.complete(true);
			return null;
		}
	}).when(slotManager).start(any(ResourceManagerId.class), any(Executor.class), any(ResourceActions.class));

	when(slotManager.registerSlotRequest(any(SlotRequest.class))).thenReturn(true);
}
 
Example 15
Source Project: flink   Source File: DispatcherTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);

	heartbeatServices = new HeartbeatServices(1000L, 10000L);

	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 
Example 16
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	configuration = new Configuration();
	haServices = new TestingHighAvailabilityServices();
	jobMasterId = JobMasterId.generate();
	jmResourceId = ResourceID.generate();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	rmLeaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);
	haServices.setResourceManagerLeaderRetriever(rmLeaderRetrievalService);

	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
}
 
Example 17
Source Project: flink   Source File: TaskExecutorSlotLifetimeTest.java    License: Apache License 2.0 6 votes vote down vote up
private TaskExecutor createTaskExecutor(Configuration configuration, TestingRpcService rpcService, TestingHighAvailabilityServices haServices, LocalUnresolvedTaskManagerLocation unresolvedTaskManagerLocation) throws IOException {
	return new TaskExecutor(
		rpcService,
		TaskManagerConfiguration.fromConfiguration(
			configuration,
			TaskExecutorResourceUtils.resourceSpecFromConfigForLocalExecution(configuration),
			InetAddress.getLoopbackAddress().getHostAddress()),
		haServices,
		new TaskManagerServicesBuilder()
			.setTaskSlotTable(TaskSlotUtils.createTaskSlotTable(1))
			.setUnresolvedTaskManagerLocation(unresolvedTaskManagerLocation)
			.build(),
		ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
		new TestingHeartbeatServices(),
		UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(),
		null,
		new BlobCacheService(
			configuration,
			new VoidBlobStore(),
			null),
		testingFatalErrorHandlerResource.getFatalErrorHandler(),
		new TestingTaskExecutorPartitionTracker(),
		TaskManagerRunner.createBackPressureSampleService(configuration, rpcService.getScheduledExecutor()));
}
 
Example 18
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws IOException {
	rpc = new TestingRpcService();

	dummyBlobCacheService = new BlobCacheService(
		new Configuration(),
		new VoidBlobStore(),
		null);

	configuration = new Configuration();

	unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
	jobId = new JobID();

	testingFatalErrorHandler = new TestingFatalErrorHandler();

	haServices = new TestingHighAvailabilityServices();
	resourceManagerLeaderRetriever = new SettableLeaderRetrievalService();
	jobManagerLeaderRetriever = new SettableLeaderRetrievalService();
	haServices.setResourceManagerLeaderRetriever(resourceManagerLeaderRetriever);
	haServices.setJobMasterLeaderRetriever(jobId, jobManagerLeaderRetriever);

	nettyShuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
}
 
Example 19
Source Project: Flink-CEPplus   Source File: DispatcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 
Example 20
Source Project: Flink-CEPplus   Source File: DispatcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private TestingDispatcher createAndStartDispatcher(HeartbeatServices heartbeatServices, TestingHighAvailabilityServices haServices, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	final TestingDispatcher dispatcher = createDispatcher(heartbeatServices, haServices, jobManagerRunnerFactory);
	dispatcher.start();

	return dispatcher;
}
 
Example 21
Source Project: Flink-CEPplus   Source File: MiniDispatcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices = new TestingHighAvailabilityServices();
	testingFatalErrorHandler = new TestingFatalErrorHandler();

	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	jobGraphFuture = new CompletableFuture<>();
	resultFuture = new CompletableFuture<>();

	testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null));
}
 
Example 22
Source Project: Flink-CEPplus   Source File: JobManagerRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
	leaderElectionService = new TestingLeaderElectionService();
	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(jobGraph.getJobID(), leaderElectionService);
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	fatalErrorHandler = new TestingFatalErrorHandler();
}
 
Example 23
Source Project: Flink-CEPplus   Source File: JobLeaderIdServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the initial job registration registers a timeout which will call
 * {@link JobLeaderIdActions#notifyJobTimeout(JobID, UUID)} when executed.
 */
@Test
public void testInitialJobTimeout() throws Exception {
	final JobID jobId = new JobID();
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);

	highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);

	ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
	Time timeout = Time.milliseconds(5000L);
	JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		scheduledExecutor,
		timeout);

	jobLeaderIdService.start(jobLeaderIdActions);

	jobLeaderIdService.addJob(jobId);

	assertTrue(jobLeaderIdService.containsJob(jobId));

	ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
	verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), anyLong(), any(TimeUnit.class));

	Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
	timeoutRunnable.run();

	ArgumentCaptor<UUID> timeoutIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);

	verify(jobLeaderIdActions, times(1)).notifyJobTimeout(eq(jobId), timeoutIdArgumentCaptor.capture());

	assertTrue(jobLeaderIdService.isValidTimeout(jobId, timeoutIdArgumentCaptor.getValue()));
}
 
Example 24
private StandaloneResourceManager createAndStartResourceManager(LeaderElectionService rmLeaderElectionService, FatalErrorHandler fatalErrorHandler) throws Exception {
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	HeartbeatServices heartbeatServices = new HeartbeatServices(1000L, HEARTBEAT_TIMEOUT);
	highAvailabilityServices.setResourceManagerLeaderElectionService(rmLeaderElectionService);

	SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		Time.minutes(5L));

	StandaloneResourceManager resourceManager =
		new StandaloneResourceManager(
			rpcService,
			ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
			resourceManagerResourceID,
			highAvailabilityServices,
			heartbeatServices,
			slotManager,
			NoOpMetricRegistry.INSTANCE,
			jobLeaderIdService,
			new ClusterInformation("localhost", 1234),
			fatalErrorHandler,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	return resourceManager;
}
 
Example 25
Source Project: Flink-CEPplus   Source File: ResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
	highAvailabilityServices = new TestingHighAvailabilityServices();
	resourceManagerLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setResourceManagerLeaderElectionService(resourceManagerLeaderElectionService);
	testingFatalErrorHandler = new TestingFatalErrorHandler();
	resourceManagerResourceId = ResourceID.generate();
}
 
Example 26
Source Project: flink   Source File: DispatcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 
Example 27
Source Project: flink   Source File: DispatcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private TestingDispatcher createAndStartDispatcher(HeartbeatServices heartbeatServices, TestingHighAvailabilityServices haServices, JobManagerRunnerFactory jobManagerRunnerFactory) throws Exception {
	final TestingDispatcher dispatcher = createDispatcher(heartbeatServices, haServices, jobManagerRunnerFactory);
	dispatcher.start();

	return dispatcher;
}
 
Example 28
Source Project: flink   Source File: MiniDispatcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws Exception {
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices = new TestingHighAvailabilityServices();
	testingFatalErrorHandler = new TestingFatalErrorHandler();

	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);

	jobGraphFuture = new CompletableFuture<>();
	resultFuture = new CompletableFuture<>();

	testingJobManagerRunnerFactory = new TestingJobManagerRunnerFactory(jobGraphFuture, resultFuture, CompletableFuture.completedFuture(null));
}
 
Example 29
Source Project: flink   Source File: JobManagerRunnerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
	leaderElectionService = new TestingLeaderElectionService();
	haServices = new TestingHighAvailabilityServices();
	haServices.setJobMasterLeaderElectionService(jobGraph.getJobID(), leaderElectionService);
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());

	fatalErrorHandler = new TestingFatalErrorHandler();
}
 
Example 30
Source Project: flink   Source File: JobLeaderIdServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the initial job registration registers a timeout which will call
 * {@link JobLeaderIdActions#notifyJobTimeout(JobID, UUID)} when executed.
 */
@Test
public void testInitialJobTimeout() throws Exception {
	final JobID jobId = new JobID();
	TestingHighAvailabilityServices highAvailabilityServices = new TestingHighAvailabilityServices();
	SettableLeaderRetrievalService leaderRetrievalService = new SettableLeaderRetrievalService(
		null,
		null);

	highAvailabilityServices.setJobMasterLeaderRetriever(jobId, leaderRetrievalService);

	ScheduledExecutor scheduledExecutor = mock(ScheduledExecutor.class);
	Time timeout = Time.milliseconds(5000L);
	JobLeaderIdActions jobLeaderIdActions = mock(JobLeaderIdActions.class);

	JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		scheduledExecutor,
		timeout);

	jobLeaderIdService.start(jobLeaderIdActions);

	jobLeaderIdService.addJob(jobId);

	assertTrue(jobLeaderIdService.containsJob(jobId));

	ArgumentCaptor<Runnable> runnableArgumentCaptor = ArgumentCaptor.forClass(Runnable.class);
	verify(scheduledExecutor).schedule(runnableArgumentCaptor.capture(), anyLong(), any(TimeUnit.class));

	Runnable timeoutRunnable = runnableArgumentCaptor.getValue();
	timeoutRunnable.run();

	ArgumentCaptor<UUID> timeoutIdArgumentCaptor = ArgumentCaptor.forClass(UUID.class);

	verify(jobLeaderIdActions, times(1)).notifyJobTimeout(eq(jobId), timeoutIdArgumentCaptor.capture());

	assertTrue(jobLeaderIdService.isValidTimeout(jobId, timeoutIdArgumentCaptor.getValue()));
}