Java Code Examples for org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway

The following examples show how to use org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: ExecutionGraphRestartTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 
Example 2
Source Project: flink   Source File: ExecutionPartitionLifecycleTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception {
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>();
	taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds))));

	final TestingShuffleMaster testingShuffleMaster = new TestingShuffleMaster();

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpJobMasterPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster);

	stateTransition1.accept(execution);
	assertFalse(releasePartitionsCallFuture.isDone());

	stateTransition2.accept(execution);
	assertTrue(releasePartitionsCallFuture.isDone());

	final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get();
	assertEquals(jobId, releasePartitionsCall.f0);
	assertThat(releasePartitionsCall.f1, contains(descriptor.getShuffleDescriptor().getResultPartitionID()));

	assertEquals(1, testingShuffleMaster.externallyReleasedPartitions.size());
	assertEquals(descriptor.getShuffleDescriptor(), testingShuffleMaster.externallyReleasedPartitions.poll());
}
 
Example 3
Source Project: flink   Source File: ExecutionGraphRestartTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(
	int numSlots, SlotPool slotPool, TaskManagerLocation taskManagerLocation) throws Exception {

	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 
Example 4
Source Project: flink   Source File: ExecutionPartitionLifecycleTest.java    License: Apache License 2.0 6 votes vote down vote up
private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception {
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>();
	taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds))));

	final TestingShuffleMaster testingShuffleMaster = new TestingShuffleMaster();

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster);

	stateTransition1.accept(execution);
	assertFalse(releasePartitionsCallFuture.isDone());

	stateTransition2.accept(execution);
	assertTrue(releasePartitionsCallFuture.isDone());

	final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get();
	assertEquals(jobId, releasePartitionsCall.f0);
	assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), releasePartitionsCall.f1);

	assertEquals(1, testingShuffleMaster.externallyReleasedPartitions.size());
	assertEquals(descriptor.getShuffleDescriptor(), testingShuffleMaster.externallyReleasedPartitions.poll());
}
 
Example 5
Source Project: flink   Source File: ExecutionGraphNotEnoughResourceTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(
		int numSlots,
		SlotPool slotPool,
		TaskManagerLocation taskManagerLocation) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final String jobManagerAddress = "foobar";
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor);
	slotPool.connectToResourceManager(resourceManagerGateway);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.registerTaskManager(taskManagerLocation.getResourceID()), mainThreadExecutor).join();

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	CompletableFuture.runAsync(() -> slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers), mainThreadExecutor).join();

	return scheduler;
}
 
Example 6
Source Project: flink   Source File: SlotPoolImplTest.java    License: Apache License 2.0 6 votes vote down vote up
private List<AllocationID> registerAndOfferSlots(TaskManagerLocation taskManagerLocation, SlotPoolImpl slotPool, int numberOfSlotsToRegister) {
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());
	final List<AllocationID> allocationIds = IntStream.range(0, numberOfSlotsToRegister)
		.mapToObj(ignored -> new AllocationID())
		.collect(Collectors.toList());

	Collection<SlotOffer> slotOffers = IntStream.range(0, numberOfSlotsToRegister)
		.mapToObj(index -> new SlotOffer(allocationIds.get(index), index, ResourceProfile.ANY))
		.collect(Collectors.toList());

	slotPool.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		slotOffers);

	return allocationIds;
}
 
Example 7
Source Project: Flink-CEPplus   Source File: AllocatedSlotsTest.java    License: Apache License 2.0 5 votes vote down vote up
private AllocatedSlot createSlot(final AllocationID allocationId, final TaskManagerLocation taskManagerLocation) {
	return new AllocatedSlot(
		allocationId,
		taskManagerLocation,
		0,
		ResourceProfile.UNKNOWN,
		new SimpleAckingTaskManagerGateway());
}
 
Example 8
Source Project: Flink-CEPplus   Source File: AvailableSlotsTest.java    License: Apache License 2.0 5 votes vote down vote up
private static AllocatedSlot createAllocatedSlot(final ResourceID resourceId) {
	return new AllocatedSlot(
		new AllocationID(),
		new TaskManagerLocation(resourceId, InetAddress.getLoopbackAddress(), 42),
		0,
		DEFAULT_TESTING_PROFILE,
		new SimpleAckingTaskManagerGateway());
}
 
Example 9
Source Project: Flink-CEPplus   Source File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	this.jobId = new JobID();

	taskManagerLocation = new LocalTaskManagerLocation();
	taskManagerGateway = new SimpleAckingTaskManagerGateway();
	resourceManagerGateway = new TestingResourceManagerGateway();
}
 
Example 10
Source Project: Flink-CEPplus   Source File: SlotSharingManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the root slot are moved from unresolved to resolved once the
 * slot context future is successfully completed
 */
@Test
public void testRootSlotTransition() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	assertTrue(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertFalse(slotSharingManager.getResolvedRootSlots().contains(rootSlot));

	// now complete the slotContextFuture
	slotContextFuture.complete(
		new SimpleSlotContext(
			new AllocationID(),
			new LocalTaskManagerLocation(),
			0,
			new SimpleAckingTaskManagerGateway()));

	assertFalse(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertTrue(slotSharingManager.getResolvedRootSlots().contains(rootSlot));
}
 
Example 11
Source Project: flink   Source File: ExecutionPartitionLifecycleTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testPartitionTrackingForStateTransition(final Consumer<Execution> stateTransition, final PartitionReleaseResult partitionReleaseResult) throws Exception {
	CompletableFuture<Tuple2<ResourceID, ResultPartitionDeploymentDescriptor>> partitionStartTrackingFuture = new CompletableFuture<>();
	CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingFuture = new CompletableFuture<>();
	CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingAndReleaseFuture = new CompletableFuture<>();
	final TestingJobMasterPartitionTracker partitionTracker = new TestingJobMasterPartitionTracker();
	partitionTracker.setStartTrackingPartitionsConsumer(
		(resourceID, resultPartitionDeploymentDescriptor) ->
			partitionStartTrackingFuture.complete(Tuple2.of(resourceID, resultPartitionDeploymentDescriptor))
	);
	partitionTracker.setStopTrackingPartitionsConsumer(partitionStopTrackingFuture::complete);
	partitionTracker.setStopTrackingAndReleasePartitionsConsumer(partitionStopTrackingAndReleaseFuture::complete);

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.BLOCKING, partitionTracker, new SimpleAckingTaskManagerGateway(), NettyShuffleMaster.INSTANCE);

	Tuple2<ResourceID, ResultPartitionDeploymentDescriptor> startTrackingCall = partitionStartTrackingFuture.get();
	assertThat(startTrackingCall.f0, equalTo(taskExecutorResourceId));
	assertThat(startTrackingCall.f1, equalTo(descriptor));

	stateTransition.accept(execution);

	switch (partitionReleaseResult) {
		case NONE:
			assertFalse(partitionStopTrackingFuture.isDone());
			assertFalse(partitionStopTrackingAndReleaseFuture.isDone());
			break;
		case STOP_TRACKING:
			assertTrue(partitionStopTrackingFuture.isDone());
			assertFalse(partitionStopTrackingAndReleaseFuture.isDone());
			final Collection<ResultPartitionID> stopTrackingCall = partitionStopTrackingFuture.get();
			assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingCall);
			break;
		case STOP_TRACKING_AND_RELEASE:
			assertFalse(partitionStopTrackingFuture.isDone());
			assertTrue(partitionStopTrackingAndReleaseFuture.isDone());
			final Collection<ResultPartitionID> stopTrackingAndReleaseCall = partitionStopTrackingAndReleaseFuture.get();
			assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingAndReleaseCall);
			break;
	}
}
 
Example 12
private SimpleSlot createSlot(TaskManagerLocation taskManagerLocation, int index) {
	return new SimpleSlot(
		mock(SlotOwner.class),
		taskManagerLocation,
		index,
		new SimpleAckingTaskManagerGateway());
}
 
Example 13
/**
 * Tests that a partially completed eager scheduling operation fails if a
 * completed slot is released. See FLINK-9099.
 */
@Test
public void testSlotReleasingFailsSchedulingOperation() throws Exception {
	final int parallelism = 2;

	final JobVertex jobVertex = new JobVertex("Testing job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setAllowQueuedScheduling(true);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);

	final SimpleSlot slot = createSlot(new SimpleAckingTaskManagerGateway(), jobGraph.getJobID(), new DummySlotOwner());
	slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot));

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	slotProvider.addSlot(jobVertex.getID(), 1, slotFuture);

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	assertThat(executionGraph.getState(), is(JobStatus.RUNNING));

	final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID());
	final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices();
	assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED));
	assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED));

	// fail the single allocated slot --> this should fail the scheduling operation
	slot.releaseSlot(new FlinkException("Test failure"));

	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED));
}
 
Example 14
@Nonnull
private static TestingLogicalSlot createTestingSlot(@Nullable CompletableFuture<?> releaseFuture) {
	return new TestingLogicalSlot(
		new LocalTaskManagerLocation(),
		new SimpleAckingTaskManagerGateway(),
		0,
		new AllocationID(),
		new SlotRequestId(),
		new SlotSharingGroupId(),
		releaseFuture);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: ExecutionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 
Example 16
Source Project: flink   Source File: ExecutionGraphSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a partially completed eager scheduling operation fails if a
 * completed slot is released. See FLINK-9099.
 */
@Test
public void testSlotReleasingFailsSchedulingOperation() throws Exception {
	final int parallelism = 2;

	final JobVertex jobVertex = new JobVertex("Testing job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);

	final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId());
	slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot));

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	slotProvider.addSlot(jobVertex.getID(), 1, slotFuture);

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	assertThat(executionGraph.getState(), is(JobStatus.RUNNING));

	final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID());
	final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices();
	assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED));
	assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED));

	// fail the single allocated slot --> this should fail the scheduling operation
	slot.releaseSlot(new FlinkException("Test failure"));

	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED));
}
 
Example 17
Source Project: Flink-CEPplus   Source File: SchedulerTestUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Instance getRandomInstance(int numSlots) {
	if (numSlots <= 0) {
		throw new IllegalArgumentException();
	}
	
	final ResourceID resourceID = ResourceID.generate();
	final InetAddress address;
	try {
		address = InetAddress.getByName("127.0.0.1");
	}
	catch (UnknownHostException e) {
		throw new RuntimeException("Test could not create IP address for localhost loopback.");
	}
	
	int dataPort = port.getAndIncrement();
	
	TaskManagerLocation ci = new TaskManagerLocation(resourceID, address, dataPort);
	
	final long GB = 1024L*1024*1024;
	HardwareDescription resources = new HardwareDescription(4, 4*GB, 3*GB, 2*GB);
	
	return new Instance(
		new SimpleAckingTaskManagerGateway(),
		ci,
		new InstanceID(),
		resources,
		numSlots);
}
 
Example 18
Source Project: flink   Source File: ExecutionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 
Example 19
Source Project: flink   Source File: SlotPoolRequestCompletionTest.java    License: Apache License 2.0 5 votes vote down vote up
private void runSlotRequestCompletionTest(
	Supplier<SlotPoolImpl> slotPoolSupplier,
	Consumer<SlotPoolImpl> actionAfterSlotRequest) throws Exception {
	try (final SlotPoolImpl slotPool = slotPoolSupplier.get()) {

		final List<SlotRequestId> slotRequestIds = IntStream.range(0, 10)
			.mapToObj(ignored -> new SlotRequestId())
			.collect(Collectors.toList());

		final List<CompletableFuture<PhysicalSlot>> slotRequests = slotRequestIds
			.stream()
			.map(slotRequestId -> slotPool.requestNewAllocatedSlot(slotRequestId, ResourceProfile.UNKNOWN, TIMEOUT))
			.collect(Collectors.toList());

		actionAfterSlotRequest.accept(slotPool);

		final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.UNKNOWN);
		final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), Collections.singleton(slotOffer));

		assertThat(acceptedSlots, containsInAnyOrder(slotOffer));

		final FlinkException testingReleaseException = new FlinkException("Testing release exception");

		// check that the slot requests get completed in sequential order
		for (int i = 0; i < slotRequestIds.size(); i++) {
			final CompletableFuture<PhysicalSlot> slotRequestFuture = slotRequests.get(i);
			slotRequestFuture.get();
			slotPool.releaseSlot(slotRequestIds.get(i), testingReleaseException);
		}
	}
}
 
Example 20
Source Project: flink   Source File: AllocatedSlotsTest.java    License: Apache License 2.0 5 votes vote down vote up
private AllocatedSlot createSlot(final AllocationID allocationId, final TaskManagerLocation taskManagerLocation) {
	return new AllocatedSlot(
		allocationId,
		taskManagerLocation,
		0,
		ResourceProfile.UNKNOWN,
		new SimpleAckingTaskManagerGateway());
}
 
Example 21
Source Project: flink   Source File: SlotPoolUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceID offerSlots(
		SlotPoolImpl slotPool,
		ComponentMainThreadExecutor mainThreadExecutor,
		List<ResourceProfile> resourceProfiles) {
	return offerSlots(
		slotPool,
		mainThreadExecutor,
		resourceProfiles,
		new SimpleAckingTaskManagerGateway());
}
 
Example 22
Source Project: flink   Source File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	this.jobId = new JobID();

	taskManagerLocation = new LocalTaskManagerLocation();
	taskManagerGateway = new SimpleAckingTaskManagerGateway();
	resourceManagerGateway = new TestingResourceManagerGateway();
}
 
Example 23
Source Project: flink   Source File: SlotSharingManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the root slot are moved from unresolved to resolved once the
 * slot context future is successfully completed
 */
@Test
public void testRootSlotTransition() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	assertTrue(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertFalse(slotSharingManager.getResolvedRootSlots().contains(rootSlot));

	// now complete the slotContextFuture
	slotContextFuture.complete(
		new SimpleSlotContext(
			new AllocationID(),
			new LocalTaskManagerLocation(),
			0,
			new SimpleAckingTaskManagerGateway()));

	assertFalse(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertTrue(slotSharingManager.getResolvedRootSlots().contains(rootSlot));
}
 
Example 24
Source Project: flink   Source File: SlotPoolSlotSharingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example 25
Source Project: flink   Source File: DefaultSchedulerTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Since checkpoint is triggered asynchronously, we need to figure out when checkpoint is really
 * triggered.
 * Note that this should be invoked before scheduler initialized.
 *
 * @return the latch representing checkpoint is really triggered
 */
private CountDownLatch getCheckpointTriggeredLatch() {
	final CountDownLatch checkpointTriggeredLatch = new CountDownLatch(1);
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	testExecutionSlotAllocator.getLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway);
	taskManagerGateway.setCheckpointConsumer(
		(executionAttemptID, jobId, checkpointId, timestamp, checkpointOptions, advanceToEndOfEventTime) -> {
			checkpointTriggeredLatch.countDown();
		});
	return checkpointTriggeredLatch;
}
 
Example 26
Source Project: flink   Source File: ExecutionPartitionLifecycleTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testPartitionTrackingForStateTransition(final Consumer<Execution> stateTransition, final PartitionReleaseResult partitionReleaseResult) throws Exception {
	CompletableFuture<Tuple2<ResourceID, ResultPartitionDeploymentDescriptor>> partitionStartTrackingFuture = new CompletableFuture<>();
	CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingFuture = new CompletableFuture<>();
	CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingAndReleaseFuture = new CompletableFuture<>();
	final TestingPartitionTracker partitionTracker = new TestingPartitionTracker();
	partitionTracker.setStartTrackingPartitionsConsumer(
		(resourceID, resultPartitionDeploymentDescriptor) ->
			partitionStartTrackingFuture.complete(Tuple2.of(resourceID, resultPartitionDeploymentDescriptor))
	);
	partitionTracker.setStopTrackingPartitionsConsumer(partitionStopTrackingFuture::complete);
	partitionTracker.setStopTrackingAndReleasePartitionsConsumer(partitionStopTrackingAndReleaseFuture::complete);

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.BLOCKING, partitionTracker, new SimpleAckingTaskManagerGateway(), NettyShuffleMaster.INSTANCE);

	Tuple2<ResourceID, ResultPartitionDeploymentDescriptor> startTrackingCall = partitionStartTrackingFuture.get();
	assertThat(startTrackingCall.f0, equalTo(taskExecutorResourceId));
	assertThat(startTrackingCall.f1, equalTo(descriptor));

	stateTransition.accept(execution);

	switch (partitionReleaseResult) {
		case NONE:
			assertFalse(partitionStopTrackingFuture.isDone());
			assertFalse(partitionStopTrackingAndReleaseFuture.isDone());
			break;
		case STOP_TRACKING:
			assertTrue(partitionStopTrackingFuture.isDone());
			assertFalse(partitionStopTrackingAndReleaseFuture.isDone());
			final Collection<ResultPartitionID> stopTrackingCall = partitionStopTrackingFuture.get();
			assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingCall);
			break;
		case STOP_TRACKING_AND_RELEASE:
			assertFalse(partitionStopTrackingFuture.isDone());
			assertTrue(partitionStopTrackingAndReleaseFuture.isDone());
			final Collection<ResultPartitionID> stopTrackingAndReleaseCall = partitionStopTrackingAndReleaseFuture.get();
			assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingAndReleaseCall);
			break;
	}
}
 
Example 27
Source Project: flink   Source File: ExecutionGraphSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a partially completed eager scheduling operation fails if a
 * completed slot is released. See FLINK-9099.
 */
@Test
public void testSlotReleasingFailsSchedulingOperation() throws Exception {
	final int parallelism = 2;

	final JobVertex jobVertex = new JobVertex("Testing job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setAllowQueuedScheduling(true);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);

	final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId());
	slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot));

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	slotProvider.addSlot(jobVertex.getID(), 1, slotFuture);

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	assertThat(executionGraph.getState(), is(JobStatus.RUNNING));

	final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID());
	final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices();
	assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED));
	assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED));

	// fail the single allocated slot --> this should fail the scheduling operation
	slot.releaseSlot(new FlinkException("Test failure"));

	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED));
}
 
Example 28
Source Project: flink   Source File: ExecutionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 
Example 29
Source Project: flink   Source File: CheckpointCoordinatorTriggeringTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTriggerCheckpointBeforePreviousOneCompleted() throws Exception {
	// create some mock Execution vertices that receive the checkpoint trigger messages
	final ExecutionAttemptID attemptID = new ExecutionAttemptID();
	final AtomicInteger taskManagerCheckpointTriggeredTimes = new AtomicInteger(0);
	final SimpleAckingTaskManagerGateway.CheckpointConsumer checkpointConsumer =
		(executionAttemptID,
		jobId, checkpointId,
		timestamp,
		checkpointOptions,
		advanceToEndOfEventTime) -> taskManagerCheckpointTriggeredTimes.incrementAndGet();
	ExecutionVertex vertex = mockExecutionVertex(attemptID, checkpointConsumer);

	// set up the coordinator and validate the initial state
	CheckpointCoordinator checkpointCoordinator = createCheckpointCoordinator(vertex);

	checkpointCoordinator.startCheckpointScheduler();
	// start a periodic checkpoint first
	final CompletableFuture<CompletedCheckpoint> onCompletionPromise1 =
		triggerPeriodicCheckpoint(checkpointCoordinator);
	assertTrue(checkpointCoordinator.isTriggering());
	assertEquals(0, checkpointCoordinator.getTriggerRequestQueue().size());
	final CompletableFuture<CompletedCheckpoint> onCompletionPromise2 =
		triggerPeriodicCheckpoint(checkpointCoordinator);
	// another trigger before the prior one finished

	assertTrue(checkpointCoordinator.isTriggering());
	assertEquals(1, checkpointCoordinator.getTriggerRequestQueue().size());

	manuallyTriggeredScheduledExecutor.triggerAll();
	assertFalse(onCompletionPromise1.isCompletedExceptionally());
	assertFalse(onCompletionPromise2.isCompletedExceptionally());
	assertFalse(checkpointCoordinator.isTriggering());
	assertEquals(0, checkpointCoordinator.getTriggerRequestQueue().size());
	assertEquals(2, taskManagerCheckpointTriggeredTimes.get());
}
 
Example 30
Source Project: flink   Source File: CheckpointCoordinatorTestingUtils.java    License: Apache License 2.0 5 votes vote down vote up
static ExecutionVertex mockExecutionVertex(
	ExecutionAttemptID attemptID,
	CheckpointConsumer checkpointConsumer) {

	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	taskManagerGateway.setCheckpointConsumer(checkpointConsumer);
	return mockExecutionVertex(attemptID, taskManagerGateway);
}