org.apache.flink.util.ExecutorUtils Java Examples

The following examples show how to use org.apache.flink.util.ExecutorUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JobStatusPollingUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailedJobResult() throws ExecutionException, InterruptedException {
	final int maxAttemptCounter = 1;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createFailedJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
		assertTrue(result.isDone() && result.get().getSerializedThrowable().isPresent());

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 
Example #2
Source File: JobStatusPollingUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testHappyPath() throws ExecutionException, InterruptedException {
	final int maxAttemptCounter = 1;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));
		assertTrue(result.isDone() && result.get().isSuccess());

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 
Example #3
Source File: JobStatusPollingUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testPolling() {
	final int maxAttemptCounter = 3;
	final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor();
	try {
		final ScheduledExecutor scheduledExecutor = new ScheduledExecutorServiceAdapter(executor);

		final CallCountingJobStatusSupplier jobStatusSupplier = new CallCountingJobStatusSupplier(maxAttemptCounter);

		final CompletableFuture<JobResult> result = JobStatusPollingUtils.pollJobResultAsync(
				jobStatusSupplier,
				() -> CompletableFuture.completedFuture(createSuccessfulJobResult(new JobID(0, 0))),
				scheduledExecutor,
				10
		);

		result.join();

		assertThat(jobStatusSupplier.getAttemptCounter(), is(equalTo(maxAttemptCounter)));

	} finally {
		ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
	}
}
 
Example #4
Source File: MiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Void> terminateExecutors(long executorShutdownTimeoutMillis) {
	synchronized (lock) {
		if (ioExecutor != null) {
			return ExecutorUtils.nonBlockingShutdown(executorShutdownTimeoutMillis, TimeUnit.MILLISECONDS, ioExecutor);
		} else {
			return CompletableFuture.completedFuture(null);
		}
	}
}
 
Example #5
Source File: MiniCluster.java    From flink with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Void> terminateExecutors(long executorShutdownTimeoutMillis) {
	synchronized (lock) {
		if (ioExecutor != null) {
			return ExecutorUtils.nonBlockingShutdown(executorShutdownTimeoutMillis, TimeUnit.MILLISECONDS, ioExecutor);
		} else {
			return CompletableFuture.completedFuture(null);
		}
	}
}
 
Example #6
Source File: LocalFileSystemTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentMkdirs() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final File root = temporaryFolder.getRoot();
	final int directoryDepth = 10;
	final int concurrentOperations = 10;

	final Collection<File> targetDirectories = createTargetDirectories(root, directoryDepth, concurrentOperations);

	final ExecutorService executor = Executors.newFixedThreadPool(concurrentOperations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentOperations);

	try {
		final Collection<CompletableFuture<Void>> mkdirsFutures = new ArrayList<>(concurrentOperations);
		for (File targetDirectory : targetDirectories) {
			final CompletableFuture<Void> mkdirsFuture = CompletableFuture.runAsync(
				() -> {
					try {
						cyclicBarrier.await();
						assertThat(fs.mkdirs(Path.fromLocalFile(targetDirectory)), is(true));
					} catch (Exception e) {
						throw new CompletionException(e);
					}
				}, executor);

			mkdirsFutures.add(mkdirsFuture);
		}

		final CompletableFuture<Void> allFutures = CompletableFuture.allOf(
			mkdirsFutures.toArray(new CompletableFuture[concurrentOperations]));

		allFutures.get();
	} finally {
		final long timeout = 10000L;
		ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #7
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we can concurrently create two {@link ActorSystem} without port conflicts.
 * This effectively tests that we don't open a socket to check for a ports availability.
 * See FLINK-10580 for more details.
 */
@Test
public void testConcurrentActorSystemCreation() throws Exception {
	final int concurrentCreations = 10;
	final ExecutorService executorService = Executors.newFixedThreadPool(concurrentCreations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentCreations);

	try {
		final List<CompletableFuture<Void>> actorSystemFutures = IntStream.range(0, concurrentCreations)
			.mapToObj(
				ignored ->
					CompletableFuture.supplyAsync(
						CheckedSupplier.unchecked(() -> {
							cyclicBarrier.await();

							return BootstrapTools.startRemoteActorSystem(
								new Configuration(),
								"localhost",
								"0",
								LOG);
						}), executorService))
			.map(
				// terminate ActorSystems
				actorSystemFuture ->
					actorSystemFuture.thenCompose(AkkaUtils::terminateActorSystem)
			).collect(Collectors.toList());

		FutureUtils.completeAll(actorSystemFutures).get();
	} finally {
		ExecutorUtils.gracefulShutdown(10000L, TimeUnit.MILLISECONDS, executorService);
	}
}
 
Example #8
Source File: CheckpointCoordinatorTriggeringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test only fails eventually.
 */
@Test
public void discardingTriggeringCheckpointWillExecuteNextCheckpointRequest() throws Exception {
	final ExecutionVertex executionVertex = mockExecutionVertex(new ExecutionAttemptID());

	final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
	final CheckpointCoordinator checkpointCoordinator = new CheckpointCoordinatorTestingUtils.CheckpointCoordinatorBuilder()
		.setTasks(new ExecutionVertex[]{executionVertex})
		.setTimer(new ScheduledExecutorServiceAdapter(scheduledExecutorService))
		.setCheckpointCoordinatorConfiguration(CheckpointCoordinatorConfiguration.builder()
			.build())
		.build();

	final CompletableFuture<String> masterHookCheckpointFuture = new CompletableFuture<>();
	final OneShotLatch triggerCheckpointLatch = new OneShotLatch();
	checkpointCoordinator.addMasterHook(new TestingMasterHook(masterHookCheckpointFuture, triggerCheckpointLatch));

	try {
		checkpointCoordinator.triggerCheckpoint(false);
		final CompletableFuture<CompletedCheckpoint> secondCheckpoint = checkpointCoordinator.triggerCheckpoint(false);

		triggerCheckpointLatch.await();
		masterHookCheckpointFuture.complete("Completed");

		// discard triggering checkpoint
		checkpointCoordinator.abortPendingCheckpoints(new CheckpointException(CheckpointFailureReason.CHECKPOINT_DECLINED));

		try {
			// verify that the second checkpoint request will be executed and eventually times out
			secondCheckpoint.get();
			fail("Expected the second checkpoint to fail.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(CheckpointException.class));
		}
	} finally {
		checkpointCoordinator.shutdown(JobStatus.FINISHED);
		ExecutorUtils.gracefulShutdown(10L, TimeUnit.SECONDS, scheduledExecutorService);
	}
}
 
Example #9
Source File: ExecutionGraphCacheTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #10
Source File: WebMonitorEndpointTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void cleansUpExpiredExecutionGraphs() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setString(RestOptions.ADDRESS, "localhost");
	configuration.setLong(WebOptions.REFRESH_INTERVAL, 5L);
	final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
	final long timeout = 10000L;

	final OneShotLatch cleanupLatch = new OneShotLatch();
	final TestingExecutionGraphCache executionGraphCache = TestingExecutionGraphCache.newBuilder()
		.setCleanupRunnable(cleanupLatch::trigger)
		.build();
	try (final WebMonitorEndpoint<RestfulGateway> webMonitorEndpoint = new WebMonitorEndpoint<>(
		RestServerEndpointConfiguration.fromConfiguration(configuration),
		CompletableFuture::new,
		configuration,
		RestHandlerConfiguration.fromConfiguration(configuration),
		CompletableFuture::new,
		NoOpTransientBlobService.INSTANCE,
		executor,
		VoidMetricFetcher.INSTANCE,
		new TestingLeaderElectionService(),
		executionGraphCache,
		new TestingFatalErrorHandler())) {

		webMonitorEndpoint.start();

		// check that the cleanup will be triggered
		cleanupLatch.await(timeout, TimeUnit.MILLISECONDS);
	} finally {
		ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #11
Source File: BootstrapToolsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we can concurrently create two {@link ActorSystem} without port conflicts.
 * This effectively tests that we don't open a socket to check for a ports availability.
 * See FLINK-10580 for more details.
 */
@Test
public void testConcurrentActorSystemCreation() throws Exception {
	final int concurrentCreations = 10;
	final ExecutorService executorService = Executors.newFixedThreadPool(concurrentCreations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentCreations);

	try {
		final List<CompletableFuture<Void>> actorSystemFutures = IntStream.range(0, concurrentCreations)
			.mapToObj(
				ignored ->
					CompletableFuture.supplyAsync(
						CheckedSupplier.unchecked(() -> {
							cyclicBarrier.await();

							return BootstrapTools.startActorSystem(
								new Configuration(),
								"localhost",
								"0",
								LOG);
						}), executorService))
			.map(
				// terminate ActorSystems
				actorSystemFuture ->
					actorSystemFuture.thenCompose(AkkaUtils::terminateActorSystem)
			).collect(Collectors.toList());

		FutureUtils.completeAll(actorSystemFutures).get();
	} finally {
		ExecutorUtils.gracefulShutdown(10000L, TimeUnit.MILLISECONDS, executorService);
	}
}
 
Example #12
Source File: DefaultExecutionGraphCacheTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new DefaultExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #13
Source File: LocalFileSystemTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentMkdirs() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final File root = temporaryFolder.getRoot();
	final int directoryDepth = 10;
	final int concurrentOperations = 10;

	final Collection<File> targetDirectories = createTargetDirectories(root, directoryDepth, concurrentOperations);

	final ExecutorService executor = Executors.newFixedThreadPool(concurrentOperations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentOperations);

	try {
		final Collection<CompletableFuture<Void>> mkdirsFutures = new ArrayList<>(concurrentOperations);
		for (File targetDirectory : targetDirectories) {
			final CompletableFuture<Void> mkdirsFuture = CompletableFuture.runAsync(
				() -> {
					try {
						cyclicBarrier.await();
						assertThat(fs.mkdirs(Path.fromLocalFile(targetDirectory)), is(true));
					} catch (Exception e) {
						throw new CompletionException(e);
					}
				}, executor);

			mkdirsFutures.add(mkdirsFuture);
		}

		final CompletableFuture<Void> allFutures = CompletableFuture.allOf(
			mkdirsFutures.toArray(new CompletableFuture[concurrentOperations]));

		allFutures.get();
	} finally {
		final long timeout = 10000L;
		ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #14
Source File: DefaultSchedulerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@After
public void tearDown() throws Exception {
	if (scheduledExecutorService != null) {
		ExecutorUtils.gracefulShutdown(TIMEOUT_MS, TimeUnit.MILLISECONDS, scheduledExecutorService);
	}

	if (executor != null) {
		ExecutorUtils.gracefulShutdown(TIMEOUT_MS, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #15
Source File: ExecutionGraphCacheTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that concurrent accesses only trigger a single AccessExecutionGraph request.
 */
@Test
public void testConcurrentAccess() throws Exception {
	final Time timeout = Time.milliseconds(100L);
	final Time timeToLive = Time.hours(1L);

	final CountingRestfulGateway restfulGateway = createCountingRestfulGateway(expectedJobId, CompletableFuture.completedFuture(expectedExecutionGraph));

	final int numConcurrentAccesses = 10;

	final ArrayList<CompletableFuture<AccessExecutionGraph>> executionGraphFutures = new ArrayList<>(numConcurrentAccesses);

	final ExecutorService executor = java.util.concurrent.Executors.newFixedThreadPool(numConcurrentAccesses);

	try (ExecutionGraphCache executionGraphCache = new ExecutionGraphCache(timeout, timeToLive)) {
		for (int i = 0; i < numConcurrentAccesses; i++) {
			CompletableFuture<AccessExecutionGraph> executionGraphFuture = CompletableFuture
				.supplyAsync(
					() -> executionGraphCache.getExecutionGraph(expectedJobId, restfulGateway),
					executor)
				.thenCompose(Function.identity());

			executionGraphFutures.add(executionGraphFuture);
		}

		final CompletableFuture<Collection<AccessExecutionGraph>> allExecutionGraphFutures = FutureUtils.combineAll(executionGraphFutures);

		Collection<AccessExecutionGraph> allExecutionGraphs = allExecutionGraphFutures.get();

		for (AccessExecutionGraph executionGraph : allExecutionGraphs) {
			assertEquals(expectedExecutionGraph, executionGraph);
		}

		assertThat(restfulGateway.getNumRequestJobCalls(), Matchers.equalTo(1));
	} finally {
		ExecutorUtils.gracefulShutdown(5000L, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #16
Source File: BootstrapToolsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we can concurrently create two {@link ActorSystem} without port conflicts.
 * This effectively tests that we don't open a socket to check for a ports availability.
 * See FLINK-10580 for more details.
 */
@Test
public void testConcurrentActorSystemCreation() throws Exception {
	final int concurrentCreations = 10;
	final ExecutorService executorService = Executors.newFixedThreadPool(concurrentCreations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentCreations);

	try {
		final List<CompletableFuture<Void>> actorSystemFutures = IntStream.range(0, concurrentCreations)
			.mapToObj(
				ignored ->
					CompletableFuture.supplyAsync(
						CheckedSupplier.unchecked(() -> {
							cyclicBarrier.await();

							return BootstrapTools.startActorSystem(
								new Configuration(),
								"localhost",
								"0",
								LOG);
						}), executorService))
			.map(
				// terminate ActorSystems
				actorSystemFuture ->
					actorSystemFuture.thenCompose(AkkaUtils::terminateActorSystem)
			).collect(Collectors.toList());

		FutureUtils.completeAll(actorSystemFutures).get();
	} finally {
		ExecutorUtils.gracefulShutdown(10000L, TimeUnit.MILLISECONDS, executorService);
	}
}
 
Example #17
Source File: LocalFileSystemTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentMkdirs() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final File root = temporaryFolder.getRoot();
	final int directoryDepth = 10;
	final int concurrentOperations = 10;

	final Collection<File> targetDirectories = createTargetDirectories(root, directoryDepth, concurrentOperations);

	final ExecutorService executor = Executors.newFixedThreadPool(concurrentOperations);
	final CyclicBarrier cyclicBarrier = new CyclicBarrier(concurrentOperations);

	try {
		final Collection<CompletableFuture<Void>> mkdirsFutures = new ArrayList<>(concurrentOperations);
		for (File targetDirectory : targetDirectories) {
			final CompletableFuture<Void> mkdirsFuture = CompletableFuture.runAsync(
				() -> {
					try {
						cyclicBarrier.await();
						assertThat(fs.mkdirs(Path.fromLocalFile(targetDirectory)), is(true));
					} catch (Exception e) {
						throw new CompletionException(e);
					}
				}, executor);

			mkdirsFutures.add(mkdirsFuture);
		}

		final CompletableFuture<Void> allFutures = CompletableFuture.allOf(
			mkdirsFutures.toArray(new CompletableFuture[concurrentOperations]));

		allFutures.get();
	} finally {
		final long timeout = 10000L;
		ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, executor);
	}
}
 
Example #18
Source File: MiniCluster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Void> terminateExecutors(long executorShutdownTimeoutMillis) {
	synchronized (lock) {
		if (ioExecutor != null) {
			return ExecutorUtils.nonBlockingShutdown(executorShutdownTimeoutMillis, TimeUnit.MILLISECONDS, ioExecutor);
		} else {
			return CompletableFuture.completedFuture(null);
		}
	}
}
 
Example #19
Source File: AsyncIOExample.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public void close() throws Exception {
	super.close();
	ExecutorUtils.gracefulShutdown(shutdownWaitTS, TimeUnit.MILLISECONDS, executorService);
}
 
Example #20
Source File: TestingComponentMainThreadExecutor.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected void after() {
	ExecutorUtils.gracefulShutdown(shutdownTimeoutMillis, TimeUnit.MILLISECONDS, innerExecutorService);
}
 
Example #21
Source File: SlotProtocolTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void afterClass() {
	ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, scheduledExecutorService);
}
 
Example #22
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the {@link SlotReport} sent to the RM does not contain
 * out dated/stale information as slots are being requested from the
 * TM.
 *
 * <p>This is a probabilistic test case and needs to be executed
 * several times to produce a failure without the fix for FLINK-12865.
 */
@Test
public void testSlotReportDoesNotContainStaleInformation() throws Exception {
	final OneShotLatch receivedSlotRequest = new OneShotLatch();
	final CompletableFuture<Void> verifySlotReportFuture = new CompletableFuture<>();
	final OneShotLatch terminateSlotReportVerification = new OneShotLatch();
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	// Assertions for this test
	testingResourceManagerGateway.setTaskExecutorHeartbeatConsumer((ignored, heartbeatPayload) -> {
		try {
			final ArrayList<SlotStatus> slots = Lists.newArrayList(heartbeatPayload.getSlotReport());
			assertThat(slots, hasSize(1));
			final SlotStatus slotStatus = slots.get(0);

			log.info("Received SlotStatus: {}", slotStatus);

			if (receivedSlotRequest.isTriggered()) {
				assertThat(slotStatus.getAllocationID(), is(notNullValue()));
			} else {
				assertThat(slotStatus.getAllocationID(), is(nullValue()));
			}
		} catch (AssertionError e) {
			verifySlotReportFuture.completeExceptionally(e);
		}

		if (terminateSlotReportVerification.isTriggered()) {
			verifySlotReportFuture.complete(null);
		}
	});
	final CompletableFuture<ResourceID> taskExecutorRegistrationFuture = new CompletableFuture<>();

	testingResourceManagerGateway.setSendSlotReportFunction(ignored -> {
		taskExecutorRegistrationFuture.complete(null);
		return CompletableFuture.completedFuture(Acknowledge.get());
	});

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder()
		.setTaskSlotTable(new AllocateSlotNotifyingTaskSlotTable(receivedSlotRequest))
		.build();
	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);
	final ResourceID taskExecutorResourceId = taskManagerServices.getUnresolvedTaskManagerLocation().getResourceID();

	taskExecutor.start();

	final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

	final ScheduledExecutorService heartbeatExecutor = java.util.concurrent.Executors.newSingleThreadScheduledExecutor();

	try {
		taskExecutorRegistrationFuture.get();

		final OneShotLatch scheduleFirstHeartbeat = new OneShotLatch();
		final ResourceID resourceManagerResourceId = testingResourceManagerGateway.getOwnResourceId();
		final long heartbeatInterval = 5L;
		heartbeatExecutor.scheduleWithFixedDelay(
			() -> {
				scheduleFirstHeartbeat.trigger();
				taskExecutorGateway.heartbeatFromResourceManager(resourceManagerResourceId);
			},
			0L,
			heartbeatInterval,
			TimeUnit.MILLISECONDS);

		scheduleFirstHeartbeat.await();

		SlotID slotId = new SlotID(taskExecutorResourceId, 0);
		final CompletableFuture<Acknowledge> requestSlotFuture = taskExecutorGateway.requestSlot(
			slotId,
			jobId,
			new AllocationID(),
			ResourceProfile.ZERO,
			"foobar",
			testingResourceManagerGateway.getFencingToken(),
			timeout);

		requestSlotFuture.get();

		terminateSlotReportVerification.trigger();

		verifySlotReportFuture.get();
	} finally {
		ExecutorUtils.gracefulShutdown(timeout.toMilliseconds(), TimeUnit.MILLISECONDS, heartbeatExecutor);
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example #23
Source File: ApplicationDispatcherBootstrapTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@After
public void cleanup() {
	ExecutorUtils.gracefulShutdown(5, TimeUnit.SECONDS, executor);
}
 
Example #24
Source File: SessionDispatcherLeaderProcessTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@AfterClass
public static void teardownClass() {
	if (ioExecutor != null) {
		ExecutorUtils.gracefulShutdown(5L, TimeUnit.SECONDS, ioExecutor);
	}
}
 
Example #25
Source File: AbstractServerBase.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Shuts down the server and all related thread pools.
 * @return A {@link CompletableFuture} that will be completed upon termination of the shutdown process.
 */
public CompletableFuture<Void> shutdownServer() {
	CompletableFuture<Void> shutdownFuture = new CompletableFuture<>();
	if (serverShutdownFuture.compareAndSet(null, shutdownFuture)) {
		log.info("Shutting down {} @ {}", serverName, serverAddress);

		final CompletableFuture<Void> groupShutdownFuture = new CompletableFuture<>();
		if (bootstrap != null) {
			EventLoopGroup group = bootstrap.group();
			if (group != null && !group.isShutdown()) {
				group.shutdownGracefully(0L, 0L, TimeUnit.MILLISECONDS)
						.addListener(finished -> {
							if (finished.isSuccess()) {
								groupShutdownFuture.complete(null);
							} else {
								groupShutdownFuture.completeExceptionally(finished.cause());
							}
						});
			} else {
				groupShutdownFuture.complete(null);
			}
		} else {
			groupShutdownFuture.complete(null);
		}

		final CompletableFuture<Void> handlerShutdownFuture = new CompletableFuture<>();
		if (handler == null) {
			handlerShutdownFuture.complete(null);
		} else {
			handler.shutdown().whenComplete((result, throwable) -> {
				if (throwable != null) {
					handlerShutdownFuture.completeExceptionally(throwable);
				} else {
					handlerShutdownFuture.complete(null);
				}
			});
		}

		final CompletableFuture<Void> queryExecShutdownFuture = CompletableFuture.runAsync(() -> {
			if (queryExecutor != null) {
				ExecutorUtils.gracefulShutdown(10L, TimeUnit.MINUTES, queryExecutor);
			}
		});

		CompletableFuture.allOf(
				queryExecShutdownFuture, groupShutdownFuture, handlerShutdownFuture
		).whenComplete((result, throwable) -> {
			if (throwable != null) {
				shutdownFuture.completeExceptionally(throwable);
			} else {
				shutdownFuture.complete(null);
			}
		});
	}
	return serverShutdownFuture.get();
}
 
Example #26
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Shuts down this registry and the associated {@link MetricReporter}.
 *
 * <p>NOTE: This operation is asynchronous and returns a future which is completed
 * once the shutdown operation has been completed.
 *
 * @return Future which is completed once the {@link MetricRegistryImpl}
 * is shut down.
 */
public CompletableFuture<Void> shutdown() {
	synchronized (lock) {
		if (isShutdown) {
			return terminationFuture;
		} else {
			isShutdown = true;
			final Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3);
			final Time gracePeriod = Time.seconds(1L);

			if (metricQueryServiceRpcService != null) {
				final CompletableFuture<Void> metricQueryServiceRpcServiceTerminationFuture = metricQueryServiceRpcService.stopService();
				terminationFutures.add(metricQueryServiceRpcServiceTerminationFuture);
			}

			Throwable throwable = null;
			for (ReporterAndSettings reporterAndSettings : reporters) {
				try {
					reporterAndSettings.getReporter().close();
				} catch (Throwable t) {
					throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
				}
			}
			reporters.clear();

			if (throwable != null) {
				terminationFutures.add(
					FutureUtils.completedExceptionally(
						new FlinkException("Could not shut down the metric reporters properly.", throwable)));
			}

			final CompletableFuture<Void> executorShutdownFuture = ExecutorUtils.nonBlockingShutdown(
				gracePeriod.toMilliseconds(),
				TimeUnit.MILLISECONDS,
				executor);

			terminationFutures.add(executorShutdownFuture);

			FutureUtils
				.completeAll(terminationFutures)
				.whenComplete(
					(Void ignored, Throwable error) -> {
						if (error != null) {
							terminationFuture.completeExceptionally(error);
						} else {
							terminationFuture.complete(null);
						}
					});

			return terminationFuture;
		}
	}
}
 
Example #27
Source File: AkkaRpcService.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public CompletableFuture<Void> closeAsync() {
	return ExecutorUtils.nonBlockingShutdown(30L, TimeUnit.SECONDS, terminationFutureExecutor);
}
 
Example #28
Source File: MetricRegistryImpl.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Shuts down this registry and the associated {@link MetricReporter}.
 *
 * <p>NOTE: This operation is asynchronous and returns a future which is completed
 * once the shutdown operation has been completed.
 *
 * @return Future which is completed once the {@link MetricRegistryImpl}
 * is shut down.
 */
public CompletableFuture<Void> shutdown() {
	synchronized (lock) {
		if (isShutdown) {
			return terminationFuture;
		} else {
			isShutdown = true;
			final Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3);
			final Time gracePeriod = Time.seconds(1L);

			if (metricQueryServiceRpcService != null) {
				final CompletableFuture<Void> metricQueryServiceRpcServiceTerminationFuture = metricQueryServiceRpcService.stopService();
				terminationFutures.add(metricQueryServiceRpcServiceTerminationFuture);
			}

			Throwable throwable = null;
			for (MetricReporter reporter : reporters) {
				try {
					reporter.close();
				} catch (Throwable t) {
					throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
				}
			}
			reporters.clear();

			if (throwable != null) {
				terminationFutures.add(
					FutureUtils.completedExceptionally(
						new FlinkException("Could not shut down the metric reporters properly.", throwable)));
			}

			final CompletableFuture<Void> executorShutdownFuture = ExecutorUtils.nonBlockingShutdown(
				gracePeriod.toMilliseconds(),
				TimeUnit.MILLISECONDS,
				executor);

			terminationFutures.add(executorShutdownFuture);

			FutureUtils
				.completeAll(terminationFutures)
				.whenComplete(
					(Void ignored, Throwable error) -> {
						if (error != null) {
							terminationFuture.completeExceptionally(error);
						} else {
							terminationFuture.complete(null);
						}
					});

			return terminationFuture;
		}
	}
}
 
Example #29
Source File: MetricRegistryImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Shuts down this registry and the associated {@link MetricReporter}.
 *
 * <p>NOTE: This operation is asynchronous and returns a future which is completed
 * once the shutdown operation has been completed.
 *
 * @return Future which is completed once the {@link MetricRegistryImpl}
 * is shut down.
 */
public CompletableFuture<Void> shutdown() {
	synchronized (lock) {
		if (isShutdown) {
			return terminationFuture;
		} else {
			isShutdown = true;
			final Collection<CompletableFuture<Void>> terminationFutures = new ArrayList<>(3);
			final Time gracePeriod = Time.seconds(1L);

			if (queryService != null) {
				final CompletableFuture<Void> queryServiceTerminationFuture = ActorUtils.nonBlockingShutDown(
					gracePeriod.toMilliseconds(),
					TimeUnit.MILLISECONDS,
					queryService);

				terminationFutures.add(queryServiceTerminationFuture);
			}

			Throwable throwable = null;
			for (MetricReporter reporter : reporters) {
				try {
					reporter.close();
				} catch (Throwable t) {
					throwable = ExceptionUtils.firstOrSuppressed(t, throwable);
				}
			}
			reporters.clear();

			if (throwable != null) {
				terminationFutures.add(
					FutureUtils.completedExceptionally(
						new FlinkException("Could not shut down the metric reporters properly.", throwable)));
			}

			final CompletableFuture<Void> executorShutdownFuture = ExecutorUtils.nonBlockingShutdown(
				gracePeriod.toMilliseconds(),
				TimeUnit.MILLISECONDS,
				executor);

			terminationFutures.add(executorShutdownFuture);

			FutureUtils
				.completeAll(terminationFutures)
				.whenComplete(
					(Void ignored, Throwable error) -> {
						if (error != null) {
							terminationFuture.completeExceptionally(error);
						} else {
							terminationFuture.complete(null);
						}
					});

			return terminationFuture;
		}
	}
}
 
Example #30
Source File: StackTraceSampleServiceTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@After
public void tearDown() throws Exception {
	if (scheduledExecutorService != null) {
		ExecutorUtils.gracefulShutdown(10, TimeUnit.SECONDS, scheduledExecutorService);
	}
}