org.apache.flink.util.function.RunnableWithException Java Examples

@Test
public void testProcessWithUnAvailableOutput() throws Exception {
	try (final MockEnvironment environment = setupEnvironment(new boolean[] {true, false})) {
		final int numberOfProcessCalls = 10;
		final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
		final StreamTask task = new MockStreamTaskBuilder(environment)
			.setStreamInputProcessor(inputProcessor)
			.build();
		final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();

		final RunnableWithException completeFutureTask = () -> {
			assertEquals(1, inputProcessor.currentNumProcessCalls);
			assertTrue(task.mailboxProcessor.isDefaultActionUnavailable());
			environment.getWriter(1).getAvailableFuture().complete(null);
		};

		executor.submit(() -> {
			executor.submit(completeFutureTask, "This task will complete the future to resume process input action."); },
			"This task will submit another task to execute after processing input once.");

		task.invoke();
		assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
	}
}
 

void runTest(RunnableWithException testMethod) throws Exception {
	if (slotManager == null) {
		WorkerResourceSpec workerResourceSpec = KubernetesWorkerResourceSpecFactory.INSTANCE
			.createDefaultWorkerResourceSpec(flinkConfig);
		slotManager = SlotManagerBuilder.newBuilder()
			.setDefaultWorkerResourceSpec(workerResourceSpec)
			.build();
		registerSlotProfile = SlotManagerImpl.generateDefaultSlotResourceProfile(workerResourceSpec, 1);
	}

	if (flinkKubeClient == null) {
		flinkKubeClient = KubernetesResourceManagerTest.this.flinkKubeClient;
	}

	resourceManager = createAndStartResourceManager(flinkConfig, slotManager, flinkKubeClient);

	try {
		testMethod.run();
	} finally {
		resourceManager.close();
	}
}
 

/**
 * Run the given action after the completion of the given future. The given future can be
 * completed normally or exceptionally. In case of an exceptional completion the, the
 * action's exception will be added to the initial exception.
 *
 * @param future to wait for its completion
 * @param runnable action which is triggered after the future's completion
 * @param executor to run the given action
 * @return Future which is completed after the action has completed. This future can contain an exception,
 * if an error occurred in the given future or action.
 */
public static CompletableFuture<Void> runAfterwardsAsync(
	CompletableFuture<?> future,
	RunnableWithException runnable,
	Executor executor) {
	final CompletableFuture<Void> resultFuture = new CompletableFuture<>();

	future.whenCompleteAsync(
		(Object ignored, Throwable throwable) -> {
			try {
				runnable.run();
			} catch (Throwable e) {
				throwable = ExceptionUtils.firstOrSuppressed(e, throwable);
			}

			if (throwable != null) {
				resultFuture.completeExceptionally(throwable);
			} else {
				resultFuture.complete(null);
			}
		},
		executor);

	return resultFuture;
}
 

/**
 * Run the given action after the completion of the given future. The given future can be
 * completed normally or exceptionally. In case of an exceptional completion the, the
 * action's exception will be added to the initial exception.
 *
 * @param future to wait for its completion
 * @param runnable action which is triggered after the future's completion
 * @param executor to run the given action
 * @return Future which is completed after the action has completed. This future can contain an exception,
 * if an error occurred in the given future or action.
 */
public static CompletableFuture<Void> runAfterwardsAsync(
	CompletableFuture<?> future,
	RunnableWithException runnable,
	Executor executor) {
	final CompletableFuture<Void> resultFuture = new CompletableFuture<>();

	future.whenCompleteAsync(
		(Object ignored, Throwable throwable) -> {
			try {
				runnable.run();
			} catch (Throwable e) {
				throwable = ExceptionUtils.firstOrSuppressed(e, throwable);
			}

			if (throwable != null) {
				resultFuture.completeExceptionally(throwable);
			} else {
				resultFuture.complete(null);
			}
		},
		executor);

	return resultFuture;
}
 

private void cleanUp() throws Exception {
	LOG.debug("cleanup, state={}", state);

	RunnableWithException[] runClose = {
			() -> sourceContext.close(),
			() -> output.close(),
			() -> format.close(),
			() -> {
				if (this.format instanceof RichInputFormat) {
					((RichInputFormat) this.format).closeInputFormat();
				}
			}};
	Exception firstException = null;

	for (RunnableWithException r : runClose) {
		try {
			r.run();
		} catch (Exception e) {
			firstException = ExceptionUtils.firstOrSuppressed(e, firstException);
		}
	}
	currentSplit = null;
	if (firstException != null) {
		throw firstException;
	}
}
 

private Future<Void> notifyCheckpointOperation(RunnableWithException runnable, String description) {
	CompletableFuture<Void> result = new CompletableFuture<>();
	mailboxProcessor.getMailboxExecutor(TaskMailbox.MAX_PRIORITY).execute(
		() -> {
			try {
				runnable.run();
			}
			catch (Exception ex) {
				result.completeExceptionally(ex);
				throw ex;
			}
			result.complete(null);
		},
		description);
	return result;
}
 

/**
 * Run the given action after the completion of the given future. The given future can be
 * completed normally or exceptionally. In case of an exceptional completion the, the
 * action's exception will be added to the initial exception.
 *
 * @param future to wait for its completion
 * @param runnable action which is triggered after the future's completion
 * @param executor to run the given action
 * @return Future which is completed after the action has completed. This future can contain an exception,
 * if an error occurred in the given future or action.
 */
public static CompletableFuture<Void> runAfterwardsAsync(
	CompletableFuture<?> future,
	RunnableWithException runnable,
	Executor executor) {
	final CompletableFuture<Void> resultFuture = new CompletableFuture<>();

	future.whenCompleteAsync(
		(Object ignored, Throwable throwable) -> {
			try {
				runnable.run();
			} catch (Throwable e) {
				throwable = ExceptionUtils.firstOrSuppressed(e, throwable);
			}

			if (throwable != null) {
				resultFuture.completeExceptionally(throwable);
			} else {
				resultFuture.complete(null);
			}
		},
		executor);

	return resultFuture;
}
 

/**
 * Sends the given <code>mail</code> using {@link TaskMailbox#putFirst(Mail)} .
 * Intended use is to control this <code>MailboxProcessor</code>; no interaction with tasks should be performed;
 */
private void sendControlMail(RunnableWithException mail, String descriptionFormat, Object... descriptionArgs) {
	mailbox.putFirst(new Mail(
			mail,
			Integer.MAX_VALUE /*not used with putFirst*/,
			descriptionFormat,
			descriptionArgs));
}
 

private void checkTaskManagerTimeoutWithCustomCanBeReleasedResponse(
		SlotManagerImpl slotManager,
		boolean canBeReleased,
		RunnableWithException doAfterCheckTriggerBeforeCanBeReleasedResponse) throws Exception {
	canBeReleasedFuture.set(new CompletableFuture<>());
	mainThreadExecutor.execute(slotManager::checkTaskManagerTimeouts); // trigger TM.canBeReleased request
	mainThreadExecutor.triggerAll();
	doAfterCheckTriggerBeforeCanBeReleasedResponse.run();
	canBeReleasedFuture.get().complete(canBeReleased); // finish TM.canBeReleased request
	mainThreadExecutor.triggerAll();
}
 

private static <T extends Throwable> void unwrappingError(Class<T> clazz, RunnableWithException r) throws Exception {
	try {
		r.run();
	} catch (Exception e) {
		throw findThrowable(e, clazz).map(te -> (Exception) te).orElse(e);
	}
}
 

/**
 * FLINK-14304: Avoid newly spawned letters to prevent input processing from ever happening.
 */
@Test
public void testAvoidStarvation() throws Exception {

	final int expectedInvocations = 3;
	final AtomicInteger counter = new AtomicInteger(0);
	MailboxThread mailboxThread = new MailboxThread() {
		@Override
		public void runDefaultAction(Controller controller) {
			if (counter.incrementAndGet() == expectedInvocations) {
				controller.allActionsCompleted();
			}
		}
	};

	mailboxThread.start();
	final MailboxProcessor mailboxProcessor = mailboxThread.getMailboxProcessor();
	final MailboxExecutor mailboxExecutor = mailboxProcessor.getMailboxExecutor(DEFAULT_PRIORITY);
	AtomicInteger index = new AtomicInteger();
	mailboxExecutor.execute(
		new RunnableWithException() {
			@Override
			public void run() {
				mailboxExecutor.execute(this, "Blocking mail" + index.incrementAndGet());
			}
		},
		"Blocking mail" + index.get());

	mailboxThread.signalStart();
	mailboxThread.join();

	Assert.assertEquals(expectedInvocations, counter.get());
	Assert.assertEquals(expectedInvocations, index.get());
}
 

private void testUnblocksInternal(
		RunnableWithException testMethod,
		Consumer<TaskMailbox> unblockMethod) throws InterruptedException {
	final Thread[] blockedThreads = new Thread[8];
	final Exception[] exceptions = new Exception[blockedThreads.length];

	CountDownLatch countDownLatch = new CountDownLatch(blockedThreads.length);

	for (int i = 0; i < blockedThreads.length; ++i) {
		final int id = i;
		Thread blocked = new Thread(() -> {
			try {
				countDownLatch.countDown();
				while (true) {
					testMethod.run();
				}
			} catch (Exception ex) {
				exceptions[id] = ex;
			}
		});
		blockedThreads[i] = blocked;
		blocked.start();
	}

	countDownLatch.await();
	unblockMethod.accept(taskMailbox);

	for (Thread blockedThread : blockedThreads) {
		blockedThread.join();
	}

	for (Exception exception : exceptions) {
		assertEquals(IllegalStateException.class, exception.getClass());
	}

}
 

private void submitInternal(ChannelStateWriteRequest request, RunnableWithException action) throws Exception {
	try {
		action.run();
	} catch (Exception ex) {
		request.cancel(ex);
		throw ex;
	}
	ensureRunning();
}
 

private void runWithChecks(RunnableWithException r) throws Exception {
	try {
		checkState(!result.isDone(), "result is already completed", result);
		r.run();
	} catch (Exception e) {
		fail(e);
		throw e;
	}
}
 

private void doComplete(boolean precondition, RunnableWithException complete, RunnableWithException... callbacks) throws Exception {
	Preconditions.checkArgument(precondition);
	complete.run();
	if (allInputsReceived && allOutputsReceived) {
		for (RunnableWithException callback : callbacks) {
			callback.run();
		}
	}
}
 

ChannelStateCheckpointWriter(
		long checkpointId,
		ChannelStateWriteResult result,
		ChannelStateSerializer serializer,
		RunnableWithException onComplete,
		CheckpointStateOutputStream checkpointStateOutputStream,
		DataOutputStream dataStream) throws Exception {
	this.checkpointId = checkpointId;
	this.result = checkNotNull(result);
	this.checkpointStream = checkNotNull(checkpointStateOutputStream);
	this.serializer = checkNotNull(serializer);
	this.dataStream = checkNotNull(dataStream);
	this.onComplete = checkNotNull(onComplete);
	runWithChecks(() -> serializer.writeHeader(dataStream));
}
 

ChannelStateCheckpointWriter(
		long checkpointId,
		ChannelStateWriteResult result,
		CheckpointStateOutputStream stream,
		ChannelStateSerializer serializer,
		RunnableWithException onComplete) throws Exception {
	this(checkpointId, result, serializer, onComplete, stream, new DataOutputStream(stream));
}
 

ChannelStateCheckpointWriter(
		CheckpointStartRequest startCheckpointItem,
		CheckpointStreamFactory streamFactory,
		ChannelStateSerializer serializer,
		RunnableWithException onComplete) throws Exception {
	this(
		startCheckpointItem.getCheckpointId(),
		startCheckpointItem.getTargetResult(),
		streamFactory.createCheckpointStateOutputStream(EXCLUSIVE),
		serializer,
		onComplete);
}
 

/**
 * A wrapper function for running test. Deal with setup and teardown logic
 * in Context.
 * @param testMethod the real test body.
 */
void runTest(RunnableWithException testMethod) throws Exception {
	startResourceManager();
	try {
		testMethod.run();
	} finally {
		stopResourceManager();
	}
}
 

@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
	if (name.startsWith("com.mapr") || name.startsWith("org.apache.hadoop")) {
		throw new ClassNotFoundException(name);
	}
	else if (name.equals(RunnableWithException.class.getName()) || name.startsWith("org.apache.log4j")) {
		return properParent.loadClass(name);
	}
	else {
		return super.loadClass(name);
	}
}
 

@Test
public void testInstantiationWhenMapRClassesAreMissing() throws Exception {
	final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRNotInClassPathTest$TestRunner";
	final ClassLoader cl = new MapRFreeClassLoader(getClass().getClassLoader());

	final RunnableWithException testRunner = Class
		.forName(testClassName, false, cl)
		.asSubclass(RunnableWithException.class)
		.newInstance();

	testRunner.run();
}
 

public static void suppressExceptions(RunnableWithException action) {
	try {
		action.run();
	}
	catch (InterruptedException e) {
		// restore interrupted state
		Thread.currentThread().interrupt();
	}
	catch (Throwable t) {
		if (isJvmFatalError(t)) {
			rethrow(t);
		}
	}
}
 

public static void suppressExceptions(RunnableWithException action) {
	try {
		action.run();
	}
	catch (InterruptedException e) {
		// restore interrupted state
		Thread.currentThread().interrupt();
	}
	catch (Throwable t) {
		if (isJvmFatalError(t)) {
			rethrow(t);
		}
	}
}
 

public static void suppressExceptions(RunnableWithException action) {
	try {
		action.run();
	}
	catch (InterruptedException e) {
		// restore interrupted state
		Thread.currentThread().interrupt();
	}
	catch (Throwable t) {
		if (isJvmFatalError(t)) {
			rethrow(t);
		}
	}
}
 

/**
 * A wrapper function for running test. Deal with setup and teardown logic
 * in Context.
 * @param testMethod the real test body.
 */
void runTest(RunnableWithException testMethod) throws Exception {
	startResourceManager();
	try {
		testMethod.run();
	} finally {
		stopResourceManager();
	}
}
 

@Test
public void testInstantiationWhenMapRClassesAreMissing() throws Exception {
	final String testClassName = "org.apache.flink.runtime.fs.maprfs.MapRNotInClassPathTest$TestRunner";
	final ClassLoader cl = new MapRFreeClassLoader(getClass().getClassLoader());

	final RunnableWithException testRunner = Class
		.forName(testClassName, false, cl)
		.asSubclass(RunnableWithException.class)
		.newInstance();

	testRunner.run();
}
 

@Override
public Class<?> loadClass(String name) throws ClassNotFoundException {
	if (name.startsWith("com.mapr") || name.startsWith("org.apache.hadoop")) {
		throw new ClassNotFoundException(name);
	}
	else if (name.equals(RunnableWithException.class.getName()) || name.startsWith("org.apache.log4j")) {
		return properParent.loadClass(name);
	}
	else {
		return super.loadClass(name);
	}
}
 

private void checkTaskManagerTimeoutWithCustomCanBeReleasedResponse(
		SlotManagerImpl slotManager,
		boolean canBeReleased,
		RunnableWithException doAfterCheckTriggerBeforeCanBeReleasedResponse) throws Exception {
	canBeReleasedFuture.set(new CompletableFuture<>());
	mainThreadExecutor.execute(slotManager::checkTaskManagerTimeouts); // trigger TM.canBeReleased request
	mainThreadExecutor.triggerAll();
	doAfterCheckTriggerBeforeCanBeReleasedResponse.run();
	canBeReleasedFuture.get().complete(canBeReleased); // finish TM.canBeReleased request
	mainThreadExecutor.triggerAll();
}
 

/**
 * A wrapper function for running test. Deal with setup and teardown logic
 * in Context.
 * @param testMethod the real test body.
 */
void runTest(RunnableWithException testMethod) throws Exception {
	startResourceManager();
	try {
		testMethod.run();
	} finally {
		stopResourceManager();
	}
}
 

/**
 * Executes the runnable, and completes {@link #resultFuture} with any exceptions thrown, during
 * its execution.
 */
private void tryWithFuture(RunnableWithException runnable) {
  try {
    runnable.run();
  } catch (Throwable t) {
    resultFuture.completeExceptionally(t);
  }
}