Java Code Examples for org.apache.flink.util.ExecutorUtils#gracefulShutdown()

The following examples show how to use org.apache.flink.util.ExecutorUtils#gracefulShutdown() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobStatusPollingUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHappyPath() throws ExecutionException, InterruptedException {
	final int maxAttemptCounter = 1;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
		assertTrue(result.isDone() && result.get().isSuccess());

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 
Example 2
Source File: JobStatusPollingUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testPolling() {
	final int maxAttemptCounter = 3;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 
Example 3
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we can concurrently create two {@link ActorSystem} without port conflicts.
 * This effectively tests that we don't open a socket to check for a ports availability.
 * See FLINK-10580 for more details.
 */
@Test
public void testConcurrentActorSystemCreation() throws Exception {
	final int concurrentCreations = 10;
	final ExecutorService executorService = Executors.newFixedThreadPool(concurrentCreations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentCreations);

	try {
		final List<CompletableFuture<Void>> actorSystemFutures = IntStream.range(0, concurrentCreations)
			.mapToObj(
				ignored ->
					CompletableFuture.supplyAsync(
						CheckedSupplier.unchecked(() -> {
							cyclicBarrier.await();

							return BootstrapTools.startActorSystem(
								new Configuration(),
								"localhost",
								"0",
								LOG);
						}), executorService))
			.map(
				// terminate ActorSystems
				actorSystemFuture ->
					actorSystemFuture.thenCompose(AkkaUtils::terminateActorSystem)
			).collect(Collectors.toList());

		FutureUtils.completeAll(actorSystemFutures).get();
	} finally {
		ExecutorUtils.gracefulShutdown(10000L, TimeUnit.MILLISECONDS, executorService);
	}
}
 
Example 4
Source File: WebMonitorEndpointTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void cleansUpExpiredExecutionGraphs() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setString(RestOptions.ADDRESS, "localhost");
	configuration.setLong(WebOptions.REFRESH_INTERVAL, 5L);
	final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
	final long timeout = 10000L;

	final OneShotLatch cleanupLatch = new OneShotLatch();
	final TestingExecutionGraphCache executionGraphCache = TestingExecutionGraphCache.newBuilder()
		.setCleanupRunnable(cleanupLatch::trigger)
		.build();
	try (final WebMonitorEndpoint<RestfulGateway> webMonitorEndpoint = new WebMonitorEndpoint<>(
		RestServerEndpointConfiguration.fromConfiguration(configuration),
		CompletableFuture::new,
		configuration,
		RestHandlerConfiguration.fromConfiguration(configuration),
		CompletableFuture::new,
		NoOpTransientBlobService.INSTANCE,
		executor,
		VoidMetricFetcher.INSTANCE,
		new TestingLeaderElectionService(),
		executionGraphCache,
		new TestingFatalErrorHandler())) {

		webMonitorEndpoint.start();

		// check that the cleanup will be triggered
		cleanupLatch.await(timeout, TimeUnit.MILLISECONDS);
	} finally {
		ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example 5
Source File: ExecutionGraphCacheTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example 6
Source File: DefaultSchedulerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@After
public void tearDown() throws Exception {
	if (scheduledExecutorService != null) {
		ExecutorUtils.gracefulShutdown(TIMEOUT_MS, TimeUnit.MILLISECONDS, scheduledExecutorService);
	}

	if (executor != null) {
		ExecutorUtils.gracefulShutdown(TIMEOUT_MS, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example 7
Source File: BootstrapToolsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we can concurrently create two {@link ActorSystem} without port conflicts.
 * This effectively tests that we don't open a socket to check for a ports availability.
 * See FLINK-10580 for more details.
 */
@Test
public void testConcurrentActorSystemCreation() throws Exception {
	final int concurrentCreations = 10;
	final ExecutorService executorService = Executors.newFixedThreadPool(concurrentCreations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentCreations);

	try {
		final List<CompletableFuture<Void>> actorSystemFutures = IntStream.range(0, concurrentCreations)
			.mapToObj(
				ignored ->
					CompletableFuture.supplyAsync(
						CheckedSupplier.unchecked(() -> {
							cyclicBarrier.await();

							return BootstrapTools.startActorSystem(
								new Configuration(),
								"localhost",
								"0",
								LOG);
						}), executorService))
			.map(
				// terminate ActorSystems
				actorSystemFuture ->
					actorSystemFuture.thenCompose(AkkaUtils::terminateActorSystem)
			).collect(Collectors.toList());

		FutureUtils.completeAll(actorSystemFutures).get();
	} finally {
		ExecutorUtils.gracefulShutdown(10000L, TimeUnit.MILLISECONDS, executorService);
	}
}
 
Example 8
Source File: AsyncIOExample.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws Exception {
	super.close();
	ExecutorUtils.gracefulShutdown(shutdownWaitTS, TimeUnit.MILLISECONDS, executorService);
}
 
Example 9
Source File: TestingComponentMainThreadExecutor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
protected void after() {
	ExecutorUtils.gracefulShutdown(shutdownTimeoutMillis, TimeUnit.MILLISECONDS, innerExecutorService);
}
 
Example 10
Source File: AsyncIOExample.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws Exception {
	super.close();
	ExecutorUtils.gracefulShutdown(shutdownWaitTS, TimeUnit.MILLISECONDS, executorService);
}
 
Example 11
Source File: AsyncIOExample.java    From flink-learning with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws Exception {
    super.close();
    ExecutorUtils.gracefulShutdown(shutdownWaitTS, TimeUnit.MILLISECONDS, executorService);
}
 
Example 12
Source File: ApplicationDispatcherBootstrapTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@After
public void cleanup() {
	ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
}
 
Example 13
Source File: SlotProtocolTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void afterClass() {
	ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, scheduledExecutorService);
}
 
Example 14
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the {@link SlotReport} sent to the RM does not contain
 * out dated/stale information as slots are being requested from the
 * TM.
 *
 * <p>This is a probabilistic test case and needs to be executed
 * several times to produce a failure without the fix for FLINK-12865.
 */
@Test
public void testSlotReportDoesNotContainStaleInformation() throws Exception {
	final OneShotLatch receivedSlotRequest = new OneShotLatch();
	final CompletableFuture<Void> verifySlotReportFuture = new CompletableFuture<>();
	final OneShotLatch terminateSlotReportVerification = new OneShotLatch();
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	// Assertions for this test
	testingResourceManagerGateway.setTaskExecutorHeartbeatConsumer((ignored, slotReport) -> {
		try {
			final ArrayList<SlotStatus> slots = Lists.newArrayList(slotReport);
			assertThat(slots, hasSize(1));
			final SlotStatus slotStatus = slots.get(0);

			log.info("Received SlotStatus: {}", slotStatus);

			if (receivedSlotRequest.isTriggered()) {
				assertThat(slotStatus.getAllocationID(), is(notNullValue()));
			} else {
				assertThat(slotStatus.getAllocationID(), is(nullValue()));
			}
		} catch (AssertionError e) {
			verifySlotReportFuture.completeExceptionally(e);
		}

		if (terminateSlotReportVerification.isTriggered()) {
			verifySlotReportFuture.complete(null);
		}
	});
	final CompletableFuture<ResourceID> taskExecutorRegistrationFuture = new CompletableFuture<>();

	testingResourceManagerGateway.setSendSlotReportFunction(ignored -> {
		taskExecutorRegistrationFuture.complete(null);
		return CompletableFuture.completedFuture(Acknowledge.get());
	});

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskSlotTable(new AllocateSlotNotifyingTaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService, receivedSlotRequest))
		.build();
	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);
	final ResourceID taskExecutorResourceId = taskManagerServices.getTaskManagerLocation().getResourceID();

	taskExecutor.start();

	final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

	final ScheduledExecutorService heartbeatExecutor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor();

	try {
		taskExecutorRegistrationFuture.get();

		final OneShotLatch scheduleFirstHeartbeat = new OneShotLatch();
		final ResourceID resourceManagerResourceId = testingResourceManagerGateway.getOwnResourceId();
		final long heartbeatInterval = 5L;
		heartbeatExecutor.scheduleWithFixedDelay(
			() -> {
				scheduleFirstHeartbeat.trigger();
				taskExecutorGateway.heartbeatFromResourceManager(resourceManagerResourceId);
			},
			0L,
			heartbeatInterval,
			TimeUnit.MILLISECONDS);

		scheduleFirstHeartbeat.await();

		SlotID slotId = new SlotID(taskExecutorResourceId, 0);
		final CompletableFuture<Acknowledge> requestSlotFuture = taskExecutorGateway.requestSlot(
			slotId,
			jobId,
			new AllocationID(),
			"foobar",
			testingResourceManagerGateway.getFencingToken(),
			timeout);

		requestSlotFuture.get();

		terminateSlotReportVerification.trigger();

		verifySlotReportFuture.get();
	} finally {
		ExecutorUtils.gracefulShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, heartbeatExecutor);
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example 15
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the {@link SlotReport} sent to the RM does not contain
 * out dated/stale information as slots are being requested from the
 * TM.
 *
 * <p>This is a probabilistic test case and needs to be executed
 * several times to produce a failure without the fix for FLINK-12865.
 */
@Test
public void testSlotReportDoesNotContainStaleInformation() throws Exception {
	final OneShotLatch receivedSlotRequest = new OneShotLatch();
	final CompletableFuture<Void> verifySlotReportFuture = new CompletableFuture<>();
	final OneShotLatch terminateSlotReportVerification = new OneShotLatch();
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	// Assertions for this test
	testingResourceManagerGateway.setTaskExecutorHeartbeatConsumer((ignored, heartbeatPayload) -> {
		try {
			final ArrayList<SlotStatus> slots = Lists.newArrayList(heartbeatPayload.getSlotReport());
			assertThat(slots, hasSize(1));
			final SlotStatus slotStatus = slots.get(0);

			log.info("Received SlotStatus: {}", slotStatus);

			if (receivedSlotRequest.isTriggered()) {
				assertThat(slotStatus.getAllocationID(), is(notNullValue()));
			} else {
				assertThat(slotStatus.getAllocationID(), is(nullValue()));
			}
		} catch (AssertionError e) {
			verifySlotReportFuture.completeExceptionally(e);
		}

		if (terminateSlotReportVerification.isTriggered()) {
			verifySlotReportFuture.complete(null);
		}
	});
	final CompletableFuture<ResourceID> taskExecutorRegistrationFuture = new CompletableFuture<>();

	testingResourceManagerGateway.setSendSlotReportFunction(ignored -> {
		taskExecutorRegistrationFuture.complete(null);
		return CompletableFuture.completedFuture(Acknowledge.get());
	});

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskSlotTable(new AllocateSlotNotifyingTaskSlotTable(receivedSlotRequest))
		.build();
	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);
	final ResourceID taskExecutorResourceId = taskManagerServices.getUnresolvedTaskManagerLocation().getResourceID();

	taskExecutor.start();

	final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

	final ScheduledExecutorService heartbeatExecutor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor();

	try {
		taskExecutorRegistrationFuture.get();

		final OneShotLatch scheduleFirstHeartbeat = new OneShotLatch();
		final ResourceID resourceManagerResourceId = testingResourceManagerGateway.getOwnResourceId();
		final long heartbeatInterval = 5L;
		heartbeatExecutor.scheduleWithFixedDelay(
			() -> {
				scheduleFirstHeartbeat.trigger();
				taskExecutorGateway.heartbeatFromResourceManager(resourceManagerResourceId);
			},
			0L,
			heartbeatInterval,
			TimeUnit.MILLISECONDS);

		scheduleFirstHeartbeat.await();

		SlotID slotId = new SlotID(taskExecutorResourceId, 0);
		final CompletableFuture<Acknowledge> requestSlotFuture = taskExecutorGateway.requestSlot(
			slotId,
			jobId,
			new AllocationID(),
			ResourceProfile.ZERO,
			"foobar",
			testingResourceManagerGateway.getFencingToken(),
			timeout);

		requestSlotFuture.get();

		terminateSlotReportVerification.trigger();

		verifySlotReportFuture.get();
	} finally {
		ExecutorUtils.gracefulShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, heartbeatExecutor);
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example 16
Source File: TaskExecutorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the {@link SlotReport} sent to the RM does not contain
 * out dated/stale information as slots are being requested from the
 * TM.
 *
 * <p>This is a probabilistic test case and needs to be executed
 * several times to produce a failure without the fix for FLINK-12865.
 */
@Test
public void testSlotReportDoesNotContainStaleInformation() throws Exception {
	final OneShotLatch receivedSlotRequest = new OneShotLatch();
	final CompletableFuture<Void> verifySlotReportFuture = new CompletableFuture<>();
	final OneShotLatch terminateSlotReportVerification = new OneShotLatch();
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	// Assertions for this test
	testingResourceManagerGateway.setTaskExecutorHeartbeatConsumer((ignored, slotReport) -> {
		try {
			final ArrayList<SlotStatus> slots = Lists.newArrayList(slotReport);
			assertThat(slots, hasSize(1));
			final SlotStatus slotStatus = slots.get(0);

			log.info("Received SlotStatus: {}", slotStatus);

			if (receivedSlotRequest.isTriggered()) {
				assertThat(slotStatus.getAllocationID(), is(notNullValue()));
			} else {
				assertThat(slotStatus.getAllocationID(), is(nullValue()));
			}
		} catch (AssertionError e) {
			verifySlotReportFuture.completeExceptionally(e);
		}

		if (terminateSlotReportVerification.isTriggered()) {
			verifySlotReportFuture.complete(null);
		}
	});
	final CompletableFuture<ResourceID> taskExecutorRegistrationFuture = new CompletableFuture<>();

	testingResourceManagerGateway.setSendSlotReportFunction(ignored -> {
		taskExecutorRegistrationFuture.complete(null);
		return CompletableFuture.completedFuture(Acknowledge.get());
	});

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskSlotTable(new AllocateSlotNotifyingTaskSlotTable(Collections.singleton(ResourceProfile.UNKNOWN), timerService, receivedSlotRequest))
		.build();
	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);
	final ResourceID taskExecutorResourceId = taskManagerServices.getTaskManagerLocation().getResourceID();

	taskExecutor.start();

	final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

	final ScheduledExecutorService heartbeatExecutor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor();

	try {
		taskExecutorRegistrationFuture.get();

		final OneShotLatch scheduleFirstHeartbeat = new OneShotLatch();
		final ResourceID resourceManagerResourceId = testingResourceManagerGateway.getOwnResourceId();
		final long heartbeatInterval = 5L;
		heartbeatExecutor.scheduleWithFixedDelay(
			() -> {
				scheduleFirstHeartbeat.trigger();
				taskExecutorGateway.heartbeatFromResourceManager(resourceManagerResourceId);
			},
			0L,
			heartbeatInterval,
			TimeUnit.MILLISECONDS);

		scheduleFirstHeartbeat.await();

		SlotID slotId = new SlotID(taskExecutorResourceId, 0);
		final CompletableFuture<Acknowledge> requestSlotFuture = taskExecutorGateway.requestSlot(
			slotId,
			jobId,
			new AllocationID(),
			"foobar",
			testingResourceManagerGateway.getFencingToken(),
			timeout);

		requestSlotFuture.get();

		terminateSlotReportVerification.trigger();

		verifySlotReportFuture.get();
	} finally {
		ExecutorUtils.gracefulShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, heartbeatExecutor);
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example 17
Source File: TestingComponentMainThreadExecutor.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected void after() {
	ExecutorUtils.gracefulShutdown(shutdownTimeoutMillis, TimeUnit.MILLISECONDS, innerExecutorService);
}
 
Example 18
Source File: AsyncIOExample.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws Exception {
	super.close();
	ExecutorUtils.gracefulShutdown(shutdownWaitTS, TimeUnit.MILLISECONDS, executorService);
}
 
Example 19
Source File: AsyncIOExample.java    From flink-learning with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws Exception {
    super.close();
    ExecutorUtils.gracefulShutdown(shutdownWaitTS, TimeUnit.MILLISECONDS, executorService);
}
 
Example 20
Source File: StackTraceSampleServiceTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
	if (scheduledExecutorService != null) {
		ExecutorUtils.gracefulShutdown(10, TimeUnit.SECONDS, scheduledExecutorService);
	}
}