Java Code Examples for org.apache.flink.runtime.testingUtils.TestingUtils#createTaskManager()

@Test
public void testTerminationOnFatalError() {
	highAvailabilityServices.setJobMasterLeaderRetriever(
		HighAvailabilityServices.DEFAULT_JOB_ID,
		new SettableLeaderRetrievalService());

	new JavaTestKit(system){{

		final ActorGateway taskManager = TestingUtils.createTaskManager(
				system,
				highAvailabilityServices, // no jobmanager
				new Configuration(),
				true,
				false);

		try {
			watch(taskManager.actor());
			taskManager.tell(new FatalError("test fatal error", new Exception("something super bad")));
			expectTerminated(d, taskManager.actor());
		}
		finally {
			taskManager.tell(Kill.getInstance());
		}
	}};
}
 

/**
 *  Tests that repeated local {@link PartitionNotFoundException}s ultimately fail the receiver.
 */
@Test
public void testLocalPartitionNotFound() throws Exception {

	new JavaTestKit(system){{

		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		final ActorGateway testActorGateway = new AkkaActorGateway(
				getTestActor(),
			LEADER_SESSION_ID);

		try {
			final IntermediateDataSetID resultId = new IntermediateDataSetID();

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
					new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			final Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);

			taskManager = TestingUtils.createTaskManager(
					system,
					highAvailabilityServices,
					config,
					true,
					true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			final JobID jid = new JobID();
			final JobVertexID vid = new JobVertexID();
			final ExecutionAttemptID eid = new ExecutionAttemptID();

			final ResultPartitionID partitionId = new ResultPartitionID();

			// Local location (on the same TM though) for the partition
			final ResultPartitionLocation loc = ResultPartitionLocation.createLocal();

			final InputChannelDeploymentDescriptor[] icdd =
					new InputChannelDeploymentDescriptor[] {
							new InputChannelDeploymentDescriptor(partitionId, loc)};

			final InputGateDeploymentDescriptor igdd =
					new InputGateDeploymentDescriptor(resultId, ResultPartitionType.PIPELINED, 0, icdd);

			final TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
					jid, "TestJob", vid, eid,
					new SerializedValue<>(new ExecutionConfig()),
					"Receiver", 1, 0, 1, 0,
					new Configuration(), new Configuration(),
					Tasks.AgnosticReceiver.class.getName(),
					Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
					Collections.singletonList(igdd),
					Collections.emptyList(),
					Collections.emptyList(), 0);

			new Within(new FiniteDuration(120, TimeUnit.SECONDS)) {
				@Override
				protected void run() {
					// Submit the task
					tm.tell(new SubmitTask(tdd), testActorGateway);
					expectMsgClass(Acknowledge.get().getClass());

					// Wait to be notified about the final execution state by the mock JM
					TaskExecutionState msg = expectMsgClass(TaskExecutionState.class);

					// The task should fail after repeated requests
					assertEquals(msg.getExecutionState(), ExecutionState.FAILED);

					Throwable error = msg.getError(getClass().getClassLoader());
					if (error.getClass() != PartitionNotFoundException.class) {
						error.printStackTrace();
						fail("Wrong exception: " + error.getMessage());
					}
				}
			};
		}
		catch (Exception e) {
			e.printStackTrace();
			fail(e.getMessage());
		}
		finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 

@Test
public void testLogNotFoundHandling() throws Exception {

	new JavaTestKit(system){{

		// we require a JobManager so that the BlobService is also started
		ActorGateway jobManager = null;
		ActorGateway taskManager = null;

		try {

			// Create the JM
			ActorRef jm = system.actorOf(Props.create(
				new SimplePartitionStateLookupJobManagerCreator(LEADER_SESSION_ID, getTestActor())));

			jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

			final int dataPort = NetUtils.getAvailablePort();
			Configuration config = new Configuration();
			config.setInteger(TaskManagerOptions.DATA_PORT, dataPort);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_INITIAL, 100);
			config.setInteger(TaskManagerOptions.NETWORK_REQUEST_BACKOFF_MAX, 200);
			config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, "/i/dont/exist");

			highAvailabilityServices.setJobMasterLeaderRetriever(
				HighAvailabilityServices.DEFAULT_JOB_ID,
				new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

			taskManager = TestingUtils.createTaskManager(
				system,
				highAvailabilityServices,
				config,
				false,
				true);

			// ---------------------------------------------------------------------------------

			final ActorGateway tm = taskManager;

			new Within(d) {
				@Override
				protected void run() {
					Future<Object> logFuture = tm.ask(TaskManagerMessages.getRequestTaskManagerLog(), timeout);
					try {
						Await.result(logFuture, timeout);
						Assert.fail();
					} catch (Exception e) {
						Assert.assertTrue(e.getMessage().startsWith("TaskManager log files are unavailable. Log file could not be found at"));
					}
				}
			};
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 

/**
 * Test that a failing schedule or update consumers call leads to the failing of the respective
 * task.
 *
 * <p>IMPORTANT: We have to make sure that the invokable's cancel method is called, because only
 * then the future is completed. We do this by not eagerly deploy consumer tasks and requiring
 * the invokable to fill one memory segment. The completed memory segment will trigger the
 * scheduling of the downstream operator since it is in pipeline mode. After we've filled the
 * memory segment, we'll block the invokable and wait for the task failure due to the failed
 * schedule or update consumers call.
 */
@Test(timeout = 10000L)
public void testFailingScheduleOrUpdateConsumersMessage() throws Exception {
	new JavaTestKit(system) {{
		final Configuration configuration = new Configuration();

		// set the memory segment to the smallest size possible, because we have to fill one
		// memory buffer to trigger the schedule or update consumers message to the downstream
		// operators
		configuration.setString(TaskManagerOptions.MEMORY_SEGMENT_SIZE, "4096");

		final JobID jid = new JobID();
		final JobVertexID vid = new JobVertexID();
		final ExecutionAttemptID eid = new ExecutionAttemptID();
		final SerializedValue<ExecutionConfig> executionConfig = new SerializedValue<>(new ExecutionConfig());

		final ResultPartitionDeploymentDescriptor resultPartitionDeploymentDescriptor = new ResultPartitionDeploymentDescriptor(
			new IntermediateDataSetID(),
			new IntermediateResultPartitionID(),
			ResultPartitionType.PIPELINED,
			1,
			1,
			true);

		final TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(jid, "TestJob", vid, eid, executionConfig,
			"TestTask", 1, 0, 1, 0, new Configuration(), new Configuration(),
			TestInvokableRecordCancel.class.getName(),
			Collections.singletonList(resultPartitionDeploymentDescriptor),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			new ArrayList<>(), Collections.emptyList(), 0);

		ActorRef jmActorRef = system.actorOf(Props.create(FailingScheduleOrUpdateConsumersJobManager.class, LEADER_SESSION_ID), "jobmanager");
		ActorGateway jobManager = new AkkaActorGateway(jmActorRef, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		final ActorGateway taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			configuration,
			true,
			true);

		try {
			TestInvokableRecordCancel.resetGotCanceledFuture();

			Future<Object> result = taskManager.ask(new SubmitTask(tdd), timeout);

			Await.result(result, timeout);

			CompletableFuture<Boolean> cancelFuture = TestInvokableRecordCancel.gotCanceled();

			assertEquals(true, cancelFuture.get());
		} finally {
			TestingUtils.stopActor(taskManager);
			TestingUtils.stopActor(jobManager);
		}
	}};
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the submit task
 * message fails.
 */
@Test
public void testSubmitTaskFailure() throws Exception {
	ActorGateway jobManager = null;
	ActorGateway taskManager = null;

	try {

		ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, LEADER_SESSION_ID));
		jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			new Configuration(),
			true,
			true);

		TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
			new JobID(),
			"test job",
			new JobVertexID(),
			new ExecutionAttemptID(),
			new SerializedValue<>(new ExecutionConfig()),
			"test task",
			0, // this will make the submission fail because the number of key groups must be >= 1
			0,
			1,
			0,
			new Configuration(),
			new Configuration(),
			"Foobar",
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			0);

		Future<Object> submitResponse = taskManager.ask(new SubmitTask(tdd), timeout);

		try {
			Await.result(submitResponse, timeout);

			fail("The submit task message should have failed.");
		} catch (IllegalArgumentException e) {
			// expected
		}
	} finally {
		TestingUtils.stopActor(jobManager);
		TestingUtils.stopActor(taskManager);
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the stop task
 * message fails.
 */
@Test
public void testStopTaskFailure() throws Exception {
	ActorGateway jobManager = null;
	ActorGateway taskManager = null;

	try {
		final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

		ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, LEADER_SESSION_ID));
		jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			new Configuration(),
			true,
			true);

		TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
			new JobID(),
			"test job",
			new JobVertexID(),
			executionAttemptId,
			new SerializedValue<>(new ExecutionConfig()),
			"test task",
			1,
			0,
			1,
			0,
			new Configuration(),
			new Configuration(),
			BlockingNoOpInvokable.class.getName(),
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			0);

		Future<Object> submitResponse = taskManager.ask(new SubmitTask(tdd), timeout);

		Await.result(submitResponse, timeout);

		final Future<Object> taskRunning = taskManager.ask(new TestingTaskManagerMessages.NotifyWhenTaskIsRunning(executionAttemptId), timeout);

		Await.result(taskRunning, timeout);

		Future<Object> stopResponse = taskManager.ask(new StopTask(executionAttemptId), timeout);

		try {
			Await.result(stopResponse, timeout);

			fail("The stop task message should have failed.");
		} catch (UnsupportedOperationException e) {
			// expected
		}
	} finally {
		TestingUtils.stopActor(jobManager);
		TestingUtils.stopActor(taskManager);
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the trigger stack
 * trace message fails.
 */
@Test
public void testStackTraceSampleFailure() throws Exception {
	ActorGateway jobManager = null;
	ActorGateway taskManager = null;

	try {

		ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, LEADER_SESSION_ID));
		jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			new Configuration(),
			true,
			true);

		Future<Object> stackTraceResponse = taskManager.ask(
			new TriggerStackTraceSample(
				0,
				new ExecutionAttemptID(),
				0,
				Time.milliseconds(1L),
				0),
			timeout);

		try {
			Await.result(stackTraceResponse, timeout);

			fail("The trigger stack trace message should have failed.");
		} catch (IllegalStateException e) {
			// expected
		}
	} finally {
		TestingUtils.stopActor(jobManager);
		TestingUtils.stopActor(taskManager);
	}
}
 

/**
 * Tests that the TaskManager sends a proper exception back to the sender if the trigger stack
 * trace message fails.
 */
@Test
public void testUpdateTaskInputPartitionsFailure() throws Exception {
	ActorGateway jobManager = null;
	ActorGateway taskManager = null;

	try {

		final ExecutionAttemptID executionAttemptId = new ExecutionAttemptID();

		ActorRef jm = system.actorOf(Props.create(SimpleJobManager.class, LEADER_SESSION_ID));
		jobManager = new AkkaActorGateway(jm, LEADER_SESSION_ID);

		highAvailabilityServices.setJobMasterLeaderRetriever(
			HighAvailabilityServices.DEFAULT_JOB_ID,
			new StandaloneLeaderRetrievalService(jobManager.path(), jobManager.leaderSessionID()));

		taskManager = TestingUtils.createTaskManager(
			system,
			highAvailabilityServices,
			new Configuration(),
			true,
			true);

		TaskDeploymentDescriptor tdd = createTaskDeploymentDescriptor(
			new JobID(),
			"test job",
			new JobVertexID(),
			executionAttemptId,
			new SerializedValue<>(new ExecutionConfig()),
			"test task",
			1,
			0,
			1,
			0,
			new Configuration(),
			new Configuration(),
			BlockingNoOpInvokable.class.getName(),
			Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
			Collections.<InputGateDeploymentDescriptor>emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			0);

		Future<Object> submitResponse = taskManager.ask(new SubmitTask(tdd), timeout);

		Await.result(submitResponse, timeout);

		Future<Object> partitionUpdateResponse = taskManager.ask(
			new TaskMessages.UpdateTaskSinglePartitionInfo(
				executionAttemptId,
				new IntermediateDataSetID(),
				new InputChannelDeploymentDescriptor(new ResultPartitionID(), ResultPartitionLocation.createLocal())),
			timeout);

		try {
			Await.result(partitionUpdateResponse, timeout);

			fail("The update task input partitions message should have failed.");
		} catch (Exception e) {
			// expected
		}
	} finally {
		TestingUtils.stopActor(jobManager);
		TestingUtils.stopActor(taskManager);
	}
}