org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway Java Examples

The following examples show how to use org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionPartitionLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception {
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>();
	taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds))));

	final TestingShuffleMaster testingShuffleMaster = new TestingShuffleMaster();

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpJobMasterPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster);

	stateTransition1.accept(execution);
	assertFalse(releasePartitionsCallFuture.isDone());

	stateTransition2.accept(execution);
	assertTrue(releasePartitionsCallFuture.isDone());

	final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get();
	assertEquals(jobId, releasePartitionsCall.f0);
	assertThat(releasePartitionsCall.f1, contains(descriptor.getShuffleDescriptor().getResultPartitionID()));

	assertEquals(1, testingShuffleMaster.externallyReleasedPartitions.size());
	assertEquals(descriptor.getShuffleDescriptor(), testingShuffleMaster.externallyReleasedPartitions.poll());
}
 
Example #2
Source File: ExecutionGraphNotEnoughResourceTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(
		int numSlots,
		SlotPool slotPool,
		TaskManagerLocation taskManagerLocation) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final String jobManagerAddress = "foobar";
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor);
	slotPool.connectToResourceManager(resourceManagerGateway);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.registerTaskManager(taskManagerLocation.getResourceID()), mainThreadExecutor).join();

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	CompletableFuture.runAsync(() -> slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers), mainThreadExecutor).join();

	return scheduler;
}
 
Example #3
Source File: SlotPoolImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private List<AllocationID> registerAndOfferSlots(TaskManagerLocation taskManagerLocation, SlotPoolImpl slotPool, int numberOfSlotsToRegister) {
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());
	final List<AllocationID> allocationIds = IntStream.range(0, numberOfSlotsToRegister)
		.mapToObj(ignored -> new AllocationID())
		.collect(Collectors.toList());

	Collection<SlotOffer> slotOffers = IntStream.range(0, numberOfSlotsToRegister)
		.mapToObj(index -> new SlotOffer(allocationIds.get(index), index, ResourceProfile.ANY))
		.collect(Collectors.toList());

	slotPool.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		slotOffers);

	return allocationIds;
}
 
Example #4
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 
Example #5
Source File: ExecutionPartitionLifecycleTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception {
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>();
	taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds))));

	final TestingShuffleMaster testingShuffleMaster = new TestingShuffleMaster();

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster);

	stateTransition1.accept(execution);
	assertFalse(releasePartitionsCallFuture.isDone());

	stateTransition2.accept(execution);
	assertTrue(releasePartitionsCallFuture.isDone());

	final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get();
	assertEquals(jobId, releasePartitionsCall.f0);
	assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), releasePartitionsCall.f1);

	assertEquals(1, testingShuffleMaster.externallyReleasedPartitions.size());
	assertEquals(descriptor.getShuffleDescriptor(), testingShuffleMaster.externallyReleasedPartitions.poll());
}
 
Example #6
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(
	int numSlots, SlotPool slotPool, TaskManagerLocation taskManagerLocation) throws Exception {

	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 
Example #7
Source File: SingleLogicalSlotTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static SlotContext createSlotContext() {
	return new SimpleSlotContext(
		new AllocationID(),
		new LocalTaskManagerLocation(),
		0,
		new SimpleAckingTaskManagerGateway(),
		ResourceProfile.ANY);
}
 
Example #8
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private SlotSharingManager.MultiTaskSlot createRootSlot(TaskManagerLocation firstTaskExecutorLocation, SlotSharingManager slotSharingManager) {
	return slotSharingManager.createRootSlot(
		new SlotRequestId(),
		CompletableFuture.completedFuture(
			new SimpleSlotContext(
				new AllocationID(),
				firstTaskExecutorLocation,
				0,
				new SimpleAckingTaskManagerGateway())),
		new SlotRequestId());
}
 
Example #9
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.ANY));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example #10
Source File: SlotPoolImplTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	this.jobId = new JobID();

	taskManagerLocation = new LocalTaskManagerLocation();
	taskManagerGateway = new SimpleAckingTaskManagerGateway();
	resourceManagerGateway = new TestingResourceManagerGateway();
}
 
Example #11
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private SimpleSlotContext createSimpleSlotContext() {
	return new SimpleSlotContext(
		new AllocationID(),
		new LocalTaskManagerLocation(),
		0,
		new SimpleAckingTaskManagerGateway());
}
 
Example #12
Source File: AvailableSlotsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private static AllocatedSlot createAllocatedSlot(final ResourceID resourceId) {
	return new AllocatedSlot(
		new AllocationID(),
		new TaskManagerLocation(resourceId, InetAddress.getLoopbackAddress(), 42),
		0,
		DEFAULT_TESTING_PROFILE,
		new SimpleAckingTaskManagerGateway());
}
 
Example #13
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetResolvedSlotWithResourceConfigured() {
	ResourceProfile rp1 = ResourceProfile.fromResources(1.0, 100);
	ResourceProfile rp2 = ResourceProfile.fromResources(2.0, 200);
	ResourceProfile allocatedSlotRp = ResourceProfile.fromResources(5.0, 500);

	SlotSharingManager slotSharingManager = createTestingSlotSharingManager();

	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
			new SlotRequestId(),
			CompletableFuture.completedFuture(
					new SimpleSlotContext(
							new AllocationID(),
							new LocalTaskManagerLocation(),
							0,
							new SimpleAckingTaskManagerGateway(),
							allocatedSlotRp)),
			new SlotRequestId());

	rootSlot.allocateSingleTaskSlot(
			new SlotRequestId(),
			rp1,
			new SlotSharingGroupId(),
			Locality.LOCAL);

	Collection<SlotSelectionStrategy.SlotInfoAndResources> resolvedRoots =
		slotSharingManager.listResolvedRootSlotInfo(new AbstractID());
	assertEquals(1, resolvedRoots.size());
	assertEquals(allocatedSlotRp.subtract(rp1), resolvedRoots.iterator().next().getRemainingResources());

	rootSlot.allocateSingleTaskSlot(
			new SlotRequestId(),
			rp2,
			new SlotSharingGroupId(),
			Locality.LOCAL);
	resolvedRoots = slotSharingManager.listResolvedRootSlotInfo(new AbstractID());
	assertEquals(1, resolvedRoots.size());
	assertEquals(allocatedSlotRp.subtract(rp1).subtract(rp2), resolvedRoots.iterator().next().getRemainingResources());
}
 
Example #14
Source File: CheckpointCoordinatorTestingUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
static ExecutionVertex mockExecutionVertex(
	ExecutionAttemptID attemptID,
	CheckpointConsumer checkpointConsumer) {

	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	taskManagerGateway.setCheckpointConsumer(checkpointConsumer);
	return mockExecutionVertex(attemptID, taskManagerGateway);
}
 
Example #15
Source File: ExecutionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 
Example #16
Source File: ExecutionGraphSchedulingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a partially completed eager scheduling operation fails if a
 * completed slot is released. See FLINK-9099.
 */
@Test
public void testSlotReleasingFailsSchedulingOperation() throws Exception {
	final int parallelism = 2;

	final JobVertex jobVertex = new JobVertex("Testing job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);

	final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId());
	slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot));

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	slotProvider.addSlot(jobVertex.getID(), 1, slotFuture);

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	assertThat(executionGraph.getState(), is(JobStatus.RUNNING));

	final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID());
	final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices();
	assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED));
	assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED));

	// fail the single allocated slot --> this should fail the scheduling operation
	slot.releaseSlot(new FlinkException("Test failure"));

	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED));
}
 
Example #17
Source File: SchedulerTestUtils.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public static Instance getRandomInstance(int numSlots) {
	if (numSlots <= 0) {
		throw new IllegalArgumentException();
	}
	
	final ResourceID resourceID = ResourceID.generate();
	final InetAddress address;
	try {
		address = InetAddress.getByName("127.0.0.1");
	}
	catch (UnknownHostException e) {
		throw new RuntimeException("Test could not create IP address for localhost loopback.");
	}
	
	int dataPort = port.getAndIncrement();
	
	TaskManagerLocation ci = new TaskManagerLocation(resourceID, address, dataPort);
	
	final long GB = 1024L*1024*1024;
	HardwareDescription resources = new HardwareDescription(4, 4*GB, 3*GB, 2*GB);
	
	return new Instance(
		new SimpleAckingTaskManagerGateway(),
		ci,
		new InstanceID(),
		resources,
		numSlots);
}
 
Example #18
Source File: SlotPoolSlotSpreadOutTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void registerTaskExecutor(int numberSlotsPerTaskExecutor) {
	final SlotPool slotPool = slotPoolResource.getSlotPool();
	final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final Collection<SlotOffer> slotOffers = IntStream
		.range(0, numberSlotsPerTaskExecutor)
		.mapToObj(index -> new SlotOffer(new AllocationID(), index, ResourceProfile.ANY))
		.collect(Collectors.toList());

	slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), slotOffers);
}
 
Example #19
Source File: ExecutionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 
Example #20
Source File: SlotPoolRequestCompletionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void runSlotRequestCompletionTest(
	Supplier<SlotPoolImpl> slotPoolSupplier,
	Consumer<SlotPoolImpl> actionAfterSlotRequest) throws Exception {
	try (final SlotPoolImpl slotPool = slotPoolSupplier.get()) {

		final List<SlotRequestId> slotRequestIds = IntStream.range(0, 10)
			.mapToObj(ignored -> new SlotRequestId())
			.collect(Collectors.toList());

		final List<CompletableFuture<PhysicalSlot>> slotRequests = slotRequestIds
			.stream()
			.map(slotRequestId -> slotPool.requestNewAllocatedSlot(slotRequestId, ResourceProfile.UNKNOWN, TIMEOUT))
			.collect(Collectors.toList());

		actionAfterSlotRequest.accept(slotPool);

		final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.UNKNOWN);
		final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), Collections.singleton(slotOffer));

		assertThat(acceptedSlots, containsInAnyOrder(slotOffer));

		final FlinkException testingReleaseException = new FlinkException("Testing release exception");

		// check that the slot requests get completed in sequential order
		for (int i = 0; i < slotRequestIds.size(); i++) {
			final CompletableFuture<PhysicalSlot> slotRequestFuture = slotRequests.get(i);
			slotRequestFuture.get();
			slotPool.releaseSlot(slotRequestIds.get(i), testingReleaseException);
		}
	}
}
 
Example #21
Source File: AllocatedSlotsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private AllocatedSlot createSlot(final AllocationID allocationId, final TaskManagerLocation taskManagerLocation) {
	return new AllocatedSlot(
		allocationId,
		taskManagerLocation,
		0,
		ResourceProfile.UNKNOWN,
		new SimpleAckingTaskManagerGateway());
}
 
Example #22
Source File: SlotPoolUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ResourceID offerSlots(
		SlotPoolImpl slotPool,
		ComponentMainThreadExecutor mainThreadExecutor,
		List<ResourceProfile> resourceProfiles) {
	return offerSlots(
		slotPool,
		mainThreadExecutor,
		resourceProfiles,
		new SimpleAckingTaskManagerGateway());
}
 
Example #23
Source File: SlotPoolImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	this.jobId = new JobID();

	taskManagerLocation = new LocalTaskManagerLocation();
	taskManagerGateway = new SimpleAckingTaskManagerGateway();
	resourceManagerGateway = new TestingResourceManagerGateway();
}
 
Example #24
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the root slot are moved from unresolved to resolved once the
 * slot context future is successfully completed
 */
@Test
public void testRootSlotTransition() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();
	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	assertTrue(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertFalse(slotSharingManager.getResolvedRootSlots().contains(rootSlot));

	// now complete the slotContextFuture
	slotContextFuture.complete(
		new SimpleSlotContext(
			new AllocationID(),
			new LocalTaskManagerLocation(),
			0,
			new SimpleAckingTaskManagerGateway()));

	assertFalse(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot));
	assertTrue(slotSharingManager.getResolvedRootSlots().contains(rootSlot));
}
 
Example #25
Source File: CheckpointCoordinatorTriggeringTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTriggerCheckpointBeforePreviousOneCompleted() throws Exception {
	// create some mock Execution vertices that receive the checkpoint trigger messages
	final ExecutionAttemptID attemptID = new ExecutionAttemptID();
	final AtomicInteger taskManagerCheckpointTriggeredTimes = new AtomicInteger(0);
	final SimpleAckingTaskManagerGateway.CheckpointConsumer checkpointConsumer =
		(executionAttemptID,
		jobId, checkpointId,
		timestamp,
		checkpointOptions,
		advanceToEndOfEventTime) -> taskManagerCheckpointTriggeredTimes.incrementAndGet();
	ExecutionVertex vertex = mockExecutionVertex(attemptID, checkpointConsumer);

	// set up the coordinator and validate the initial state
	CheckpointCoordinator checkpointCoordinator = createCheckpointCoordinator(vertex);

	checkpointCoordinator.startCheckpointScheduler();
	// start a periodic checkpoint first
	final CompletableFuture<CompletedCheckpoint> onCompletionPromise1 =
		triggerPeriodicCheckpoint(checkpointCoordinator);
	assertTrue(checkpointCoordinator.isTriggering());
	assertEquals(0, checkpointCoordinator.getTriggerRequestQueue().size());
	final CompletableFuture<CompletedCheckpoint> onCompletionPromise2 =
		triggerPeriodicCheckpoint(checkpointCoordinator);
	// another trigger before the prior one finished

	assertTrue(checkpointCoordinator.isTriggering());
	assertEquals(1, checkpointCoordinator.getTriggerRequestQueue().size());

	manuallyTriggeredScheduledExecutor.triggerAll();
	assertFalse(onCompletionPromise1.isCompletedExceptionally());
	assertFalse(onCompletionPromise2.isCompletedExceptionally());
	assertFalse(checkpointCoordinator.isTriggering());
	assertEquals(0, checkpointCoordinator.getTriggerRequestQueue().size());
	assertEquals(2, taskManagerCheckpointTriggeredTimes.get());
}
 
Example #26
Source File: ExecutionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a slot release will atomically release the assigned {@link Execution}.
 */
@Test
public void testSlotReleaseAtomicallyReleasesExecution() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot(
		slotOwner,
		new SimpleAckingTaskManagerGateway(),
		new SlotRequestId());
	final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot);

	final CountDownLatch slotRequestLatch = new CountDownLatch(1);
	final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> {
		slotRequestLatch.countDown();
		return slotFuture;
	});
	final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt();

	executionGraph.start(testMainThreadUtil.getMainThreadExecutor());
	testMainThreadUtil.execute(executionGraph::scheduleForExecution);

	// wait until the slot has been requested
	slotRequestLatch.await();

	testMainThreadUtil.execute(() -> {
		assertThat(execution.getAssignedResource(), is(sameInstance(slot)));

		slot.release(new FlinkException("Test exception"));

		assertThat(execution.getReleaseFuture().isDone(), is(true));
	});
}
 
Example #27
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example #28
Source File: ExecutionGraphSchedulingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a partially completed eager scheduling operation fails if a
 * completed slot is released. See FLINK-9099.
 */
@Test
public void testSlotReleasingFailsSchedulingOperation() throws Exception {
	final int parallelism = 2;

	final JobVertex jobVertex = new JobVertex("Testing job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setAllowQueuedScheduling(true);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);

	final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId());
	slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot));

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	slotProvider.addSlot(jobVertex.getID(), 1, slotFuture);

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	assertThat(executionGraph.getState(), is(JobStatus.RUNNING));

	final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID());
	final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices();
	assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED));
	assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED));

	// fail the single allocated slot --> this should fail the scheduling operation
	slot.releaseSlot(new FlinkException("Test failure"));

	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED));
}
 
Example #29
Source File: DefaultSchedulerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Since checkpoint is triggered asynchronously, we need to figure out when checkpoint is really
 * triggered.
 * Note that this should be invoked before scheduler initialized.
 *
 * @return the latch representing checkpoint is really triggered
 */
private CountDownLatch getCheckpointTriggeredLatch() {
	final CountDownLatch checkpointTriggeredLatch = new CountDownLatch(1);
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	testExecutionSlotAllocator.getLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway);
	taskManagerGateway.setCheckpointConsumer(
		(executionAttemptID, jobId, checkpointId, timestamp, checkpointOptions, advanceToEndOfEventTime) -> {
			checkpointTriggeredLatch.countDown();
		});
	return checkpointTriggeredLatch;
}
 
Example #30
Source File: ExecutionPartitionLifecycleTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testPartitionTrackingForStateTransition(final Consumer<Execution> stateTransition, final PartitionReleaseResult partitionReleaseResult) throws Exception {
	CompletableFuture<Tuple2<ResourceID, ResultPartitionDeploymentDescriptor>> partitionStartTrackingFuture = new CompletableFuture<>();
	CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingFuture = new CompletableFuture<>();
	CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingAndReleaseFuture = new CompletableFuture<>();
	final TestingPartitionTracker partitionTracker = new TestingPartitionTracker();
	partitionTracker.setStartTrackingPartitionsConsumer(
		(resourceID, resultPartitionDeploymentDescriptor) ->
			partitionStartTrackingFuture.complete(Tuple2.of(resourceID, resultPartitionDeploymentDescriptor))
	);
	partitionTracker.setStopTrackingPartitionsConsumer(partitionStopTrackingFuture::complete);
	partitionTracker.setStopTrackingAndReleasePartitionsConsumer(partitionStopTrackingAndReleaseFuture::complete);

	setupExecutionGraphAndStartRunningJob(ResultPartitionType.BLOCKING, partitionTracker, new SimpleAckingTaskManagerGateway(), NettyShuffleMaster.INSTANCE);

	Tuple2<ResourceID, ResultPartitionDeploymentDescriptor> startTrackingCall = partitionStartTrackingFuture.get();
	assertThat(startTrackingCall.f0, equalTo(taskExecutorResourceId));
	assertThat(startTrackingCall.f1, equalTo(descriptor));

	stateTransition.accept(execution);

	switch (partitionReleaseResult) {
		case NONE:
			assertFalse(partitionStopTrackingFuture.isDone());
			assertFalse(partitionStopTrackingAndReleaseFuture.isDone());
			break;
		case STOP_TRACKING:
			assertTrue(partitionStopTrackingFuture.isDone());
			assertFalse(partitionStopTrackingAndReleaseFuture.isDone());
			final Collection<ResultPartitionID> stopTrackingCall = partitionStopTrackingFuture.get();
			assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingCall);
			break;
		case STOP_TRACKING_AND_RELEASE:
			assertFalse(partitionStopTrackingFuture.isDone());
			assertTrue(partitionStopTrackingAndReleaseFuture.isDone());
			final Collection<ResultPartitionID> stopTrackingAndReleaseCall = partitionStopTrackingAndReleaseFuture.get();
			assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingAndReleaseCall);
			break;
	}
}