Java Code Examples for org.apache.flink.runtime.operators.testutils.MockEnvironment

The following examples show how to use org.apache.flink.runtime.operators.testutils.MockEnvironment. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
SubtaskCheckpointCoordinator build() throws IOException {
	if (environment == null) {
		this.environment = MockEnvironment.builder().build();
	}
	if (checkpointStorage == null) {
		this.checkpointStorage = new MemoryBackendCheckpointStorage(environment.getJobID(), null, null, 1024);
	}
	if (asyncExceptionHandler == null) {
		this.asyncExceptionHandler = new NonHandleAsyncException();
	}

	return new SubtaskCheckpointCoordinatorImpl(
		checkpointStorage,
		taskName,
		actionExecutor,
		closeableRegistry,
		executorService,
		environment,
		asyncExceptionHandler,
		unalignedCheckpointEnabled,
		prepareInputSnapshot,
		maxRecordAbortedCheckpoints);
}
 
Example 2
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example 3
/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example 4
Source Project: Flink-CEPplus   Source File: SourceFunctionUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static <T extends Serializable> List<T> runRichSourceFunction(SourceFunction<T> sourceFunction) throws Exception {
	try (MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("MockTask")
				.setMemorySize(3 * 1024 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1024)
				.build()) {

		AbstractStreamOperator<?> operator = mock(AbstractStreamOperator.class);
		when(operator.getExecutionConfig()).thenReturn(new ExecutionConfig());

		RuntimeContext runtimeContext = new StreamingRuntimeContext(
			operator,
			environment,
			new HashMap<>());
		((RichFunction) sourceFunction).setRuntimeContext(runtimeContext);
		((RichFunction) sourceFunction).open(new Configuration());

		return runNonRichSourceFunction(sourceFunction);
	}
}
 
Example 5
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test checks the async exceptions handling wraps the message and cause as an AsynchronousException
 * and propagates this to the environment.
 */
@Test
public void streamTaskAsyncExceptionHandler_handleException_forwardsMessageProperly() {
	MockEnvironment mockEnvironment = MockEnvironment.builder().build();
	RuntimeException expectedException = new RuntimeException("RUNTIME EXCEPTION");

	final StreamTask.StreamTaskAsyncExceptionHandler asyncExceptionHandler = new StreamTask.StreamTaskAsyncExceptionHandler(mockEnvironment);

	mockEnvironment.setExpectedExternalFailureCause(AsynchronousException.class);
	final String expectedErrorMessage = "EXPECTED_ERROR MESSAGE";

	asyncExceptionHandler.handleAsyncException(expectedErrorMessage, expectedException);

	// expect an AsynchronousException containing the supplied error details
	Optional<? extends Throwable> actualExternalFailureCause = mockEnvironment.getActualExternalFailureCause();
	final Throwable actualException = actualExternalFailureCause
		.orElseThrow(() -> new AssertionError("Expected exceptional completion"));

	assertThat(actualException, instanceOf(AsynchronousException.class));
	assertThat(actualException.getMessage(), is("EXPECTED_ERROR MESSAGE"));
	assertThat(actualException.getCause(), is(expectedException));
}
 
Example 6
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testExecuteMailboxActionsAfterLeavingInputProcessorMailboxLoop() throws Exception {
	OneShotLatch latch = new OneShotLatch();
	try (MockEnvironment mockEnvironment = new MockEnvironmentBuilder().build()) {
		RunningTask<StreamTask<?, ?>> task = runTask(() -> new StreamTask<Object, StreamOperator<Object>>(mockEnvironment) {
			@Override
			protected void init() throws Exception {
			}

			@Override
			protected void processInput(MailboxDefaultAction.Controller controller) throws Exception {
				mailboxProcessor.getMailboxExecutor(0).execute(latch::trigger, "trigger");
				controller.allActionsCompleted();
			}
		});
		latch.await();
		task.waitForTaskCompletion(false);
	}
}
 
Example 7
/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example 8
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example 9
/**
 * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via
 * the configuration.
 */
@Test
public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(0);

		setupSourceOperator(operator, executionConfig, env, timeProvider);
	});
}
 
Example 10
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testProcessWithUnAvailableOutput() throws Exception {
	try (final MockEnvironment environment = setupEnvironment(new boolean[] {true, false})) {
		final int numberOfProcessCalls = 10;
		final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
		final StreamTask task = new MockStreamTaskBuilder(environment)
			.setStreamInputProcessor(inputProcessor)
			.build();
		final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();

		final RunnableWithException completeFutureTask = () -> {
			assertEquals(1, inputProcessor.currentNumProcessCalls);
			assertTrue(task.mailboxProcessor.isDefaultActionUnavailable());
			environment.getWriter(1).getAvailableFuture().complete(null);
		};

		executor.submit(() -> {
			executor.submit(completeFutureTask, "This task will complete the future to resume process input action."); },
			"This task will submit another task to execute after processing input once.");

		task.invoke();
		assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
	}
}
 
Example 11
Source Project: flink   Source File: CollectSinkFunctionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
	ioManager = new IOManagerAsync();
	MockEnvironment environment = new MockEnvironmentBuilder()
		.setTaskName("mockTask")
		.setManagedMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
		.setIOManager(ioManager)
		.build();
	runtimeContext = new MockStreamingRuntimeContext(false, 1, 0, environment);

	gateway = new MockOperatorEventGateway();
	coordinator = new CollectSinkOperatorCoordinator(SOCKET_TIMEOUT_MILLIS);
	coordinator.start();

	// only used in checkpointed tests
	functionInitializationContext = new MockFunctionInitializationContext();

	jobFinished = false;
}
 
Example 12
Source Project: flink   Source File: SubtaskCheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNotifyCheckpointComplete() throws Exception {
	TestTaskStateManager stateManager = new TestTaskStateManager();
	MockEnvironment mockEnvironment = MockEnvironment.builder().setTaskStateManager(stateManager).build();
	SubtaskCheckpointCoordinator subtaskCheckpointCoordinator = new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.build();

	final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);

	long checkpointId = 42L;
	{
		subtaskCheckpointCoordinator.notifyCheckpointComplete(checkpointId, operatorChain, () -> true);
		assertEquals(checkpointId, stateManager.getNotifiedCompletedCheckpointId());
	}

	long newCheckpointId = checkpointId + 1;
	{
		subtaskCheckpointCoordinator.notifyCheckpointComplete(newCheckpointId, operatorChain, () -> false);
		// even task is not running, state manager could still receive the notification.
		assertEquals(newCheckpointId, stateManager.getNotifiedCompletedCheckpointId());
	}
}
 
Example 13
Source Project: flink   Source File: SubtaskCheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNotifyCheckpointAbortedManyTimes() throws Exception {
	MockEnvironment mockEnvironment = MockEnvironment.builder().build();
	int maxRecordAbortedCheckpoints = 256;
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.setMaxRecordAbortedCheckpoints(maxRecordAbortedCheckpoints)
		.build();

	final OperatorChain<?, ?> operatorChain = getOperatorChain(mockEnvironment);

	long notifyAbortedTimes = maxRecordAbortedCheckpoints + 42;
	for (int i = 1; i < notifyAbortedTimes; i++) {
		subtaskCheckpointCoordinator.notifyCheckpointAborted(i, operatorChain, () -> true);
		assertEquals(Math.min(maxRecordAbortedCheckpoints, i), subtaskCheckpointCoordinator.getAbortedCheckpointSize());
	}
}
 
Example 14
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
private MockEnvironment setupEnvironment(boolean[] outputAvailabilities) {
	final Configuration configuration = new Configuration();
	new MockStreamConfig(configuration, outputAvailabilities.length);

	final List<ResultPartitionWriter> writers = new ArrayList<>(outputAvailabilities.length);
	for (int i = 0; i < outputAvailabilities.length; i++) {
		writers.add(new AvailabilityTestResultPartitionWriter(outputAvailabilities[i]));
	}

	final MockEnvironment environment = new MockEnvironmentBuilder()
		.setTaskConfiguration(configuration)
		.build();
	environment.addOutputs(writers);
	return environment;
}
 
Example 15
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	BlockingCloseStreamOperator.resetLatches();
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setManagedMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {

		RunningTask<StreamTask<Void, BlockingCloseStreamOperator>> task = runTask(() -> new NoOpStreamTask<>(mockEnvironment));

		BlockingCloseStreamOperator.inClose.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(task.streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.finishClose.trigger();

		task.waitForTaskCompletion(false);

		// now the StreamTask should no longer be running
		assertFalse(task.streamTask.isRunning());
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: AsyncWaitOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
	long timeout = 2000L;

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncWaitFunction,
		TIMEOUT,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 
Example 17
Source Project: Flink-CEPplus   Source File: AsyncWaitOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testTimeoutExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	AsyncFunction<Integer, Integer> asyncFunction = new NoOpAsyncFunction<>();
	long timeout = 10L; // 1 milli second

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncFunction,
		timeout,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	harness.setProcessingTime(10L);

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}
}
 
Example 18
Source Project: Flink-CEPplus   Source File: OperatorChainTest.java    License: Apache License 2.0 5 votes vote down vote up
@SafeVarargs
private static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(
		OneInputStreamOperator<T, T>... operators) {

	checkNotNull(operators);
	checkArgument(operators.length > 0);

	try (MockEnvironment env = MockEnvironment.builder().build()) {

	final StreamTask<?, ?> containingTask = new OneInputStreamTask<T, OneInputStreamOperator<T, T>>(env);

		final StreamStatusProvider statusProvider = mock(StreamStatusProvider.class);
		final StreamConfig cfg = new StreamConfig(new Configuration());

		final StreamOperator<?>[] ops = new StreamOperator<?>[operators.length];

		// initial output goes to nowhere
		@SuppressWarnings({"unchecked", "rawtypes"})
		WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(
				new Output[0], statusProvider);

		// build the reverse operators array
		for (int i = 0; i < ops.length; i++) {
			OneInputStreamOperator<T, T> op = operators[ops.length - i - 1];
			op.setup(containingTask, cfg, lastWriter);
			lastWriter = new ChainingOutput<>(op, statusProvider, null);
			ops[i] = op;
		}

		@SuppressWarnings("unchecked")
		final OP head = (OP) operators[0];

		return new OperatorChain<>(
				ops,
				new RecordWriterOutput<?>[0],
				lastWriter,
				head);
	}
}
 
Example 19
Source Project: flink   Source File: SubtaskCheckpointCoordinatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNotifyCheckpointAbortedDuringAsyncPhase() throws Exception {
	MockEnvironment mockEnvironment = MockEnvironment.builder().build();
	SubtaskCheckpointCoordinatorImpl subtaskCheckpointCoordinator = (SubtaskCheckpointCoordinatorImpl) new MockSubtaskCheckpointCoordinatorBuilder()
		.setEnvironment(mockEnvironment)
		.setExecutor(Executors.newSingleThreadExecutor())
		.setUnalignedCheckpointEnabled(true)
		.build();

	final BlockingRunnableFuture rawKeyedStateHandleFuture = new BlockingRunnableFuture();
	OperatorSnapshotFutures operatorSnapshotResult = new OperatorSnapshotFutures(
		DoneFuture.of(SnapshotResult.empty()),
		rawKeyedStateHandleFuture,
		DoneFuture.of(SnapshotResult.empty()),
		DoneFuture.of(SnapshotResult.empty()),
		DoneFuture.of(SnapshotResult.empty()),
		DoneFuture.of(SnapshotResult.empty()));

	final OperatorChain<String, AbstractStreamOperator<String>> operatorChain = operatorChain(new CheckpointOperator(operatorSnapshotResult));

	long checkpointId = 42L;
	subtaskCheckpointCoordinator.getChannelStateWriter().start(checkpointId, CheckpointOptions.forCheckpointWithDefaultLocation());
	subtaskCheckpointCoordinator.checkpointState(
		new CheckpointMetaData(checkpointId, System.currentTimeMillis()),
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		new CheckpointMetrics(),
		operatorChain,
		() -> true);
	rawKeyedStateHandleFuture.awaitRun();
	assertEquals(1, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
	assertFalse(rawKeyedStateHandleFuture.isCancelled());

	subtaskCheckpointCoordinator.notifyCheckpointAborted(checkpointId, operatorChain, () -> true);
	assertTrue(rawKeyedStateHandleFuture.isCancelled());
	assertEquals(0, subtaskCheckpointCoordinator.getAsyncCheckpointRunnableSize());
}
 
Example 20
/**
 * Verifies that latency metrics can be enabled via the configuration.
 */
@Test
public void testLatencyMarkEmissionEnabledViaFlinkConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		Configuration tmConfig = new Configuration();
		tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval);

		Environment env = MockEnvironment.builder()
			.setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig))
			.build();

		setupSourceOperator(operator, new ExecutionConfig(), env, timeProvider);
	});
}
 
Example 21
/**
 * Verifies that by default no latency metrics are emitted.
 */
@Test
public void testLatencyMarkEmissionDisabled() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		setupSourceOperator(operator, new ExecutionConfig(), MockEnvironment.builder().build(), timeProvider);
	});
}
 
Example 22
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig}.
 */
@Test
public void testLatencyMarkEmissionEnabledViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		setupSourceOperator(operator, executionConfig, MockEnvironment.builder().build(), timeProvider);
	});
}
 
Example 23
Source Project: Flink-CEPplus   Source File: StreamOperatorChainingTest.java    License: Apache License 2.0 5 votes vote down vote up
private MockEnvironment createMockEnvironment(String taskName) {
	return new MockEnvironmentBuilder()
		.setTaskName(taskName)
		.setMemorySize(3 * 1024 * 1024)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(1024)
		.build();
}
 
Example 24
public KeyedOneInputStreamOperatorTestHarness(
		final OneInputStreamOperator<IN, OUT> operator,
		final  KeySelector<IN, K> keySelector,
		final TypeInformation<K> keyType,
		final MockEnvironment environment) throws Exception {

	super(operator, environment);

	ClosureCleaner.clean(keySelector, ExecutionConfig.ClosureCleanerLevel.RECURSIVE, false);
	config.setStatePartitioner(0, keySelector);
	config.setStateKeySerializer(keyType.createSerializer(executionConfig));
}
 
Example 25
Source Project: flink   Source File: AsyncWaitOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private MockEnvironment createMockEnvironment() {
	return new MockEnvironmentBuilder()
		.setTaskName("foobarTask")
		.setMemorySize(1024 * 1024L)
		.setInputSplitProvider(new MockInputSplitProvider())
		.setBufferSize(4 * 1024)
		.build();
}
 
Example 26
Source Project: flink   Source File: AsyncWaitOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testUserExceptionHandling(AsyncDataStream.OutputMode outputMode) throws Exception {
	UserExceptionAsyncFunction asyncWaitFunction = new UserExceptionAsyncFunction();
	long timeout = 2000L;

	AsyncWaitOperator<Integer, Integer> asyncWaitOperator = new AsyncWaitOperator<>(
		asyncWaitFunction,
		TIMEOUT,
		2,
		outputMode);

	final MockEnvironment mockEnvironment = createMockEnvironment();
	mockEnvironment.setExpectedExternalFailureCause(Throwable.class);

	OneInputStreamOperatorTestHarness<Integer, Integer> harness = new OneInputStreamOperatorTestHarness<>(
		asyncWaitOperator,
		IntSerializer.INSTANCE,
		mockEnvironment);

	harness.open();

	synchronized (harness.getCheckpointLock()) {
		harness.processElement(1, 1L);
	}

	synchronized (harness.getCheckpointLock()) {
		harness.close();
	}

	assertTrue(harness.getEnvironment().getActualExternalFailureCause().isPresent());
}
 
Example 27
Source Project: flink   Source File: InputFormatSourceFunctionTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testFormatLifecycle(final boolean midCancel) throws Exception {

		final int noOfSplits = 5;
		final int cancelAt = 2;

		final LifeCycleTestInputFormat format = new LifeCycleTestInputFormat();
		final InputFormatSourceFunction<Integer> reader = new InputFormatSourceFunction<>(format, TypeInformation.of(Integer.class));

		try (MockEnvironment environment =
				new MockEnvironmentBuilder()
					.setTaskName("no")
					.setMemorySize(4 * MemoryManager.DEFAULT_PAGE_SIZE)
					.build()) {

			reader.setRuntimeContext(new MockRuntimeContext(format, noOfSplits, environment));

			Assert.assertTrue(!format.isConfigured);
			Assert.assertTrue(!format.isInputFormatOpen);
			Assert.assertTrue(!format.isSplitOpen);

			reader.open(new Configuration());
			Assert.assertTrue(format.isConfigured);

			TestSourceContext ctx = new TestSourceContext(reader, format, midCancel, cancelAt);
			reader.run(ctx);

			int splitsSeen = ctx.getSplitsSeen();
			Assert.assertTrue(midCancel ? splitsSeen == cancelAt : splitsSeen == noOfSplits);

			// we have exhausted the splits so the
			// format and splits should be closed by now

			Assert.assertTrue(!format.isSplitOpen);
			Assert.assertTrue(!format.isInputFormatOpen);
		}
	}
 
Example 28
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the StreamTask first closes all of its operators before setting its
 * state to not running (isRunning == false)
 *
 * <p>See FLINK-7430.
 */
@Test
public void testOperatorClosingBeforeStopRunning() throws Throwable {
	BlockingCloseStreamOperator.resetLatches();
	Configuration taskConfiguration = new Configuration();
	StreamConfig streamConfig = new StreamConfig(taskConfiguration);
	streamConfig.setStreamOperator(new BlockingCloseStreamOperator());
	streamConfig.setOperatorID(new OperatorID());

	try (MockEnvironment mockEnvironment =
			new MockEnvironmentBuilder()
				.setTaskName("Test Task")
				.setMemorySize(32L * 1024L)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(1)
				.setTaskConfiguration(taskConfiguration)
				.build()) {

		RunningTask<StreamTask<Void, BlockingCloseStreamOperator>> task = runTask(() -> new NoOpStreamTask<>(mockEnvironment));

		BlockingCloseStreamOperator.inClose.await();

		// check that the StreamTask is not yet in isRunning == false
		assertTrue(task.streamTask.isRunning());

		// let the operator finish its close operation
		BlockingCloseStreamOperator.finishClose.trigger();

		task.waitForTaskCompletion(false);

		// now the StreamTask should no longer be running
		assertFalse(task.streamTask.isRunning());
	}
}
 
Example 29
/**
 * Verifies that by default no latency metrics are emitted.
 */
@Test
public void testLatencyMarkEmissionDisabled() throws Exception {
	testLatencyMarkEmission(0, (operator, timeProvider) -> {
		setupSourceOperator(operator, new ExecutionConfig(), MockEnvironment.builder().build(), timeProvider);
	});
}
 
Example 30
/**
 * Verifies that latency metrics can be enabled via the {@link ExecutionConfig}.
 */
@Test
public void testLatencyMarkEmissionEnabledViaExecutionConfig() throws Exception {
	testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> {
		ExecutionConfig executionConfig = new ExecutionConfig();
		executionConfig.setLatencyTrackingInterval(latencyMarkInterval);

		setupSourceOperator(operator, executionConfig, MockEnvironment.builder().build(), timeProvider);
	});
}