org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway Java Examples
The following examples show how to use
org.apache.flink.runtime.executiongraph.utils.SimpleAckingTaskManagerGateway.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionPartitionLifecycleTest.java From flink with Apache License 2.0 | 6 votes |
private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception { final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>(); taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds)))); final TestingShuffleMaster testingShuffleMaster = new TestingShuffleMaster(); setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpJobMasterPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster); stateTransition1.accept(execution); assertFalse(releasePartitionsCallFuture.isDone()); stateTransition2.accept(execution); assertTrue(releasePartitionsCallFuture.isDone()); final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get(); assertEquals(jobId, releasePartitionsCall.f0); assertThat(releasePartitionsCall.f1, contains(descriptor.getShuffleDescriptor().getResultPartitionID())); assertEquals(1, testingShuffleMaster.externallyReleasedPartitions.size()); assertEquals(descriptor.getShuffleDescriptor(), testingShuffleMaster.externallyReleasedPartitions.poll()); }
Example #2
Source File: ExecutionGraphNotEnoughResourceTest.java From flink with Apache License 2.0 | 6 votes |
private static Scheduler createSchedulerWithSlots( int numSlots, SlotPool slotPool, TaskManagerLocation taskManagerLocation) throws Exception { final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); final String jobManagerAddress = "foobar"; final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway(); slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor); slotPool.connectToResourceManager(resourceManagerGateway); Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool); scheduler.start(mainThreadExecutor); CompletableFuture.runAsync(() -> slotPool.registerTaskManager(taskManagerLocation.getResourceID()), mainThreadExecutor).join(); final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS); for (int i = 0; i < numSlots; i++) { final AllocationID allocationId = new AllocationID(); final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY); slotOffers.add(slotOffer); } CompletableFuture.runAsync(() -> slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers), mainThreadExecutor).join(); return scheduler; }
Example #3
Source File: SlotPoolImplTest.java From flink with Apache License 2.0 | 6 votes |
private List<AllocationID> registerAndOfferSlots(TaskManagerLocation taskManagerLocation, SlotPoolImpl slotPool, int numberOfSlotsToRegister) { slotPool.registerTaskManager(taskManagerLocation.getResourceID()); final List<AllocationID> allocationIds = IntStream.range(0, numberOfSlotsToRegister) .mapToObj(ignored -> new AllocationID()) .collect(Collectors.toList()); Collection<SlotOffer> slotOffers = IntStream.range(0, numberOfSlotsToRegister) .mapToObj(index -> new SlotOffer(allocationIds.get(index), index, ResourceProfile.ANY)) .collect(Collectors.toList()); slotPool.offerSlots( taskManagerLocation, new SimpleAckingTaskManagerGateway(), slotOffers); return allocationIds; }
Example #4
Source File: ExecutionGraphRestartTest.java From flink with Apache License 2.0 | 6 votes |
private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception { final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); setupSlotPool(slotPool); Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool); scheduler.start(mainThreadExecutor); slotPool.registerTaskManager(taskManagerLocation.getResourceID()); final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS); for (int i = 0; i < numSlots; i++) { final AllocationID allocationId = new AllocationID(); final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY); slotOffers.add(slotOffer); } slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers); return scheduler; }
Example #5
Source File: ExecutionPartitionLifecycleTest.java From flink with Apache License 2.0 | 6 votes |
private void testPartitionReleaseOnStateTransitionsAfterRunning(Consumer<Execution> stateTransition1, Consumer<Execution> stateTransition2) throws Exception { final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); final CompletableFuture<Tuple2<JobID, Collection<ResultPartitionID>>> releasePartitionsCallFuture = new CompletableFuture<>(); taskManagerGateway.setReleasePartitionsConsumer(((jobID, partitionIds) -> releasePartitionsCallFuture.complete(Tuple2.of(jobID, partitionIds)))); final TestingShuffleMaster testingShuffleMaster = new TestingShuffleMaster(); setupExecutionGraphAndStartRunningJob(ResultPartitionType.PIPELINED, NoOpPartitionTracker.INSTANCE, taskManagerGateway, testingShuffleMaster); stateTransition1.accept(execution); assertFalse(releasePartitionsCallFuture.isDone()); stateTransition2.accept(execution); assertTrue(releasePartitionsCallFuture.isDone()); final Tuple2<JobID, Collection<ResultPartitionID>> releasePartitionsCall = releasePartitionsCallFuture.get(); assertEquals(jobId, releasePartitionsCall.f0); assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), releasePartitionsCall.f1); assertEquals(1, testingShuffleMaster.externallyReleasedPartitions.size()); assertEquals(descriptor.getShuffleDescriptor(), testingShuffleMaster.externallyReleasedPartitions.poll()); }
Example #6
Source File: ExecutionGraphRestartTest.java From flink with Apache License 2.0 | 6 votes |
private static Scheduler createSchedulerWithSlots( int numSlots, SlotPool slotPool, TaskManagerLocation taskManagerLocation) throws Exception { final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); setupSlotPool(slotPool); Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, slotPool); scheduler.start(mainThreadExecutor); slotPool.registerTaskManager(taskManagerLocation.getResourceID()); final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS); for (int i = 0; i < numSlots; i++) { final AllocationID allocationId = new AllocationID(); final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY); slotOffers.add(slotOffer); } slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers); return scheduler; }
Example #7
Source File: SingleLogicalSlotTest.java From flink with Apache License 2.0 | 5 votes |
private static SlotContext createSlotContext() { return new SimpleSlotContext( new AllocationID(), new LocalTaskManagerLocation(), 0, new SimpleAckingTaskManagerGateway(), ResourceProfile.ANY); }
Example #8
Source File: SlotSharingManagerTest.java From flink with Apache License 2.0 | 5 votes |
private SlotSharingManager.MultiTaskSlot createRootSlot(TaskManagerLocation firstTaskExecutorLocation, SlotSharingManager slotSharingManager) { return slotSharingManager.createRootSlot( new SlotRequestId(), CompletableFuture.completedFuture( new SimpleSlotContext( new AllocationID(), firstTaskExecutorLocation, 0, new SimpleAckingTaskManagerGateway())), new SlotRequestId()); }
Example #9
Source File: SlotPoolSlotSharingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSingleQueuedSharedSlotScheduling() throws Exception { final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>(); final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway(); testingResourceManagerGateway.setRequestSlotConsumer( (SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId())); LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation(); final SlotPoolImpl slotPool = slotPoolResource.getSlotPool(); slotPool.registerTaskManager(taskManagerLocation.getResourceID()); SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId(); final SlotProvider slotProvider = slotPoolResource.getSlotProvider(); CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot( new ScheduledUnit( new JobVertexID(), slotSharingGroupId, null), SlotProfile.noRequirements(), TestingUtils.infiniteTime()); assertFalse(logicalSlotFuture.isDone()); final AllocationID allocationId = allocationIdFuture.get(); boolean booleanCompletableFuture = slotPool.offerSlot( taskManagerLocation, new SimpleAckingTaskManagerGateway(), new SlotOffer( allocationId, 0, ResourceProfile.ANY)); assertTrue(booleanCompletableFuture); final LogicalSlot logicalSlot = logicalSlotFuture.get(); assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId()); }
Example #10
Source File: SlotPoolImplTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { this.jobId = new JobID(); taskManagerLocation = new LocalTaskManagerLocation(); taskManagerGateway = new SimpleAckingTaskManagerGateway(); resourceManagerGateway = new TestingResourceManagerGateway(); }
Example #11
Source File: SlotSharingManagerTest.java From flink with Apache License 2.0 | 5 votes |
private SimpleSlotContext createSimpleSlotContext() { return new SimpleSlotContext( new AllocationID(), new LocalTaskManagerLocation(), 0, new SimpleAckingTaskManagerGateway()); }
Example #12
Source File: AvailableSlotsTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static AllocatedSlot createAllocatedSlot(final ResourceID resourceId) { return new AllocatedSlot( new AllocationID(), new TaskManagerLocation(resourceId, InetAddress.getLoopbackAddress(), 42), 0, DEFAULT_TESTING_PROFILE, new SimpleAckingTaskManagerGateway()); }
Example #13
Source File: SlotSharingManagerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetResolvedSlotWithResourceConfigured() { ResourceProfile rp1 = ResourceProfile.fromResources(1.0, 100); ResourceProfile rp2 = ResourceProfile.fromResources(2.0, 200); ResourceProfile allocatedSlotRp = ResourceProfile.fromResources(5.0, 500); SlotSharingManager slotSharingManager = createTestingSlotSharingManager(); SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot( new SlotRequestId(), CompletableFuture.completedFuture( new SimpleSlotContext( new AllocationID(), new LocalTaskManagerLocation(), 0, new SimpleAckingTaskManagerGateway(), allocatedSlotRp)), new SlotRequestId()); rootSlot.allocateSingleTaskSlot( new SlotRequestId(), rp1, new SlotSharingGroupId(), Locality.LOCAL); Collection<SlotSelectionStrategy.SlotInfoAndResources> resolvedRoots = slotSharingManager.listResolvedRootSlotInfo(new AbstractID()); assertEquals(1, resolvedRoots.size()); assertEquals(allocatedSlotRp.subtract(rp1), resolvedRoots.iterator().next().getRemainingResources()); rootSlot.allocateSingleTaskSlot( new SlotRequestId(), rp2, new SlotSharingGroupId(), Locality.LOCAL); resolvedRoots = slotSharingManager.listResolvedRootSlotInfo(new AbstractID()); assertEquals(1, resolvedRoots.size()); assertEquals(allocatedSlotRp.subtract(rp1).subtract(rp2), resolvedRoots.iterator().next().getRemainingResources()); }
Example #14
Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0 | 5 votes |
static ExecutionVertex mockExecutionVertex( ExecutionAttemptID attemptID, CheckpointConsumer checkpointConsumer) { final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); taskManagerGateway.setCheckpointConsumer(checkpointConsumer); return mockExecutionVertex(attemptID, taskManagerGateway); }
Example #15
Source File: ExecutionTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests that a slot release will atomically release the assigned {@link Execution}. */ @Test public void testSlotReleaseAtomicallyReleasesExecution() throws Exception { final JobVertex jobVertex = createNoOpJobVertex(); final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner(); final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot( slotOwner, new SimpleAckingTaskManagerGateway(), new SlotRequestId()); final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot); final CountDownLatch slotRequestLatch = new CountDownLatch(1); final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> { slotRequestLatch.countDown(); return slotFuture; }); final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph( new JobID(), slotProvider, new NoRestartStrategy(), jobVertex); final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt(); executionGraph.start(testMainThreadUtil.getMainThreadExecutor()); testMainThreadUtil.execute(executionGraph::scheduleForExecution); // wait until the slot has been requested slotRequestLatch.await(); testMainThreadUtil.execute(() -> { assertThat(execution.getAssignedResource(), is(sameInstance(slot))); slot.release(new FlinkException("Test exception")); assertThat(execution.getReleaseFuture().isDone(), is(true)); }); }
Example #16
Source File: ExecutionGraphSchedulingTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that a partially completed eager scheduling operation fails if a * completed slot is released. See FLINK-9099. */ @Test public void testSlotReleasingFailsSchedulingOperation() throws Exception { final int parallelism = 2; final JobVertex jobVertex = new JobVertex("Testing job vertex"); jobVertex.setInvokableClass(NoOpInvokable.class); jobVertex.setParallelism(parallelism); final JobGraph jobGraph = new JobGraph(jobVertex); jobGraph.setScheduleMode(ScheduleMode.EAGER); final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism); final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId()); slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot)); final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>(); slotProvider.addSlot(jobVertex.getID(), 1, slotFuture); final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider); executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread()); executionGraph.scheduleForExecution(); assertThat(executionGraph.getState(), is(JobStatus.RUNNING)); final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID()); final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices(); assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED)); assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED)); // fail the single allocated slot --> this should fail the scheduling operation slot.releaseSlot(new FlinkException("Test failure")); assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED)); }
Example #17
Source File: SchedulerTestUtils.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static Instance getRandomInstance(int numSlots) { if (numSlots <= 0) { throw new IllegalArgumentException(); } final ResourceID resourceID = ResourceID.generate(); final InetAddress address; try { address = InetAddress.getByName("127.0.0.1"); } catch (UnknownHostException e) { throw new RuntimeException("Test could not create IP address for localhost loopback."); } int dataPort = port.getAndIncrement(); TaskManagerLocation ci = new TaskManagerLocation(resourceID, address, dataPort); final long GB = 1024L*1024*1024; HardwareDescription resources = new HardwareDescription(4, 4*GB, 3*GB, 2*GB); return new Instance( new SimpleAckingTaskManagerGateway(), ci, new InstanceID(), resources, numSlots); }
Example #18
Source File: SlotPoolSlotSpreadOutTest.java From flink with Apache License 2.0 | 5 votes |
private void registerTaskExecutor(int numberSlotsPerTaskExecutor) { final SlotPool slotPool = slotPoolResource.getSlotPool(); final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation(); slotPool.registerTaskManager(taskManagerLocation.getResourceID()); final Collection<SlotOffer> slotOffers = IntStream .range(0, numberSlotsPerTaskExecutor) .mapToObj(index -> new SlotOffer(new AllocationID(), index, ResourceProfile.ANY)) .collect(Collectors.toList()); slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), slotOffers); }
Example #19
Source File: ExecutionTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that a slot release will atomically release the assigned {@link Execution}. */ @Test public void testSlotReleaseAtomicallyReleasesExecution() throws Exception { final JobVertex jobVertex = createNoOpJobVertex(); final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner(); final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot( slotOwner, new SimpleAckingTaskManagerGateway(), new SlotRequestId()); final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot); final CountDownLatch slotRequestLatch = new CountDownLatch(1); final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> { slotRequestLatch.countDown(); return slotFuture; }); final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph( slotProvider, new NoRestartStrategy(), jobVertex); final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt(); executionGraph.start(testMainThreadUtil.getMainThreadExecutor()); testMainThreadUtil.execute(executionGraph::scheduleForExecution); // wait until the slot has been requested slotRequestLatch.await(); testMainThreadUtil.execute(() -> { assertThat(execution.getAssignedResource(), is(sameInstance(slot))); slot.release(new FlinkException("Test exception")); assertThat(execution.getReleaseFuture().isDone(), is(true)); }); }
Example #20
Source File: SlotPoolRequestCompletionTest.java From flink with Apache License 2.0 | 5 votes |
private void runSlotRequestCompletionTest( Supplier<SlotPoolImpl> slotPoolSupplier, Consumer<SlotPoolImpl> actionAfterSlotRequest) throws Exception { try (final SlotPoolImpl slotPool = slotPoolSupplier.get()) { final List<SlotRequestId> slotRequestIds = IntStream.range(0, 10) .mapToObj(ignored -> new SlotRequestId()) .collect(Collectors.toList()); final List<CompletableFuture<PhysicalSlot>> slotRequests = slotRequestIds .stream() .map(slotRequestId -> slotPool.requestNewAllocatedSlot(slotRequestId, ResourceProfile.UNKNOWN, TIMEOUT)) .collect(Collectors.toList()); actionAfterSlotRequest.accept(slotPool); final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation(); slotPool.registerTaskManager(taskManagerLocation.getResourceID()); final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.UNKNOWN); final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), Collections.singleton(slotOffer)); assertThat(acceptedSlots, containsInAnyOrder(slotOffer)); final FlinkException testingReleaseException = new FlinkException("Testing release exception"); // check that the slot requests get completed in sequential order for (int i = 0; i < slotRequestIds.size(); i++) { final CompletableFuture<PhysicalSlot> slotRequestFuture = slotRequests.get(i); slotRequestFuture.get(); slotPool.releaseSlot(slotRequestIds.get(i), testingReleaseException); } } }
Example #21
Source File: AllocatedSlotsTest.java From flink with Apache License 2.0 | 5 votes |
private AllocatedSlot createSlot(final AllocationID allocationId, final TaskManagerLocation taskManagerLocation) { return new AllocatedSlot( allocationId, taskManagerLocation, 0, ResourceProfile.UNKNOWN, new SimpleAckingTaskManagerGateway()); }
Example #22
Source File: SlotPoolUtils.java From flink with Apache License 2.0 | 5 votes |
public static ResourceID offerSlots( SlotPoolImpl slotPool, ComponentMainThreadExecutor mainThreadExecutor, List<ResourceProfile> resourceProfiles) { return offerSlots( slotPool, mainThreadExecutor, resourceProfiles, new SimpleAckingTaskManagerGateway()); }
Example #23
Source File: SlotPoolImplTest.java From flink with Apache License 2.0 | 5 votes |
@Before public void setUp() throws Exception { this.jobId = new JobID(); taskManagerLocation = new LocalTaskManagerLocation(); taskManagerGateway = new SimpleAckingTaskManagerGateway(); resourceManagerGateway = new TestingResourceManagerGateway(); }
Example #24
Source File: SlotSharingManagerTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the root slot are moved from unresolved to resolved once the * slot context future is successfully completed */ @Test public void testRootSlotTransition() { final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions(); SlotSharingManager slotSharingManager = new SlotSharingManager( SLOT_SHARING_GROUP_ID, allocatedSlotActions, SLOT_OWNER); CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>(); SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot( new SlotRequestId(), slotContextFuture, new SlotRequestId()); assertTrue(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot)); assertFalse(slotSharingManager.getResolvedRootSlots().contains(rootSlot)); // now complete the slotContextFuture slotContextFuture.complete( new SimpleSlotContext( new AllocationID(), new LocalTaskManagerLocation(), 0, new SimpleAckingTaskManagerGateway())); assertFalse(slotSharingManager.getUnresolvedRootSlots().contains(rootSlot)); assertTrue(slotSharingManager.getResolvedRootSlots().contains(rootSlot)); }
Example #25
Source File: CheckpointCoordinatorTriggeringTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTriggerCheckpointBeforePreviousOneCompleted() throws Exception { // create some mock Execution vertices that receive the checkpoint trigger messages final ExecutionAttemptID attemptID = new ExecutionAttemptID(); final AtomicInteger taskManagerCheckpointTriggeredTimes = new AtomicInteger(0); final SimpleAckingTaskManagerGateway.CheckpointConsumer checkpointConsumer = (executionAttemptID, jobId, checkpointId, timestamp, checkpointOptions, advanceToEndOfEventTime) -> taskManagerCheckpointTriggeredTimes.incrementAndGet(); ExecutionVertex vertex = mockExecutionVertex(attemptID, checkpointConsumer); // set up the coordinator and validate the initial state CheckpointCoordinator checkpointCoordinator = createCheckpointCoordinator(vertex); checkpointCoordinator.startCheckpointScheduler(); // start a periodic checkpoint first final CompletableFuture<CompletedCheckpoint> onCompletionPromise1 = triggerPeriodicCheckpoint(checkpointCoordinator); assertTrue(checkpointCoordinator.isTriggering()); assertEquals(0, checkpointCoordinator.getTriggerRequestQueue().size()); final CompletableFuture<CompletedCheckpoint> onCompletionPromise2 = triggerPeriodicCheckpoint(checkpointCoordinator); // another trigger before the prior one finished assertTrue(checkpointCoordinator.isTriggering()); assertEquals(1, checkpointCoordinator.getTriggerRequestQueue().size()); manuallyTriggeredScheduledExecutor.triggerAll(); assertFalse(onCompletionPromise1.isCompletedExceptionally()); assertFalse(onCompletionPromise2.isCompletedExceptionally()); assertFalse(checkpointCoordinator.isTriggering()); assertEquals(0, checkpointCoordinator.getTriggerRequestQueue().size()); assertEquals(2, taskManagerCheckpointTriggeredTimes.get()); }
Example #26
Source File: ExecutionTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that a slot release will atomically release the assigned {@link Execution}. */ @Test public void testSlotReleaseAtomicallyReleasesExecution() throws Exception { final JobVertex jobVertex = createNoOpJobVertex(); final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner(); final SingleLogicalSlot slot = ExecutionGraphSchedulingTest.createSingleLogicalSlot( slotOwner, new SimpleAckingTaskManagerGateway(), new SlotRequestId()); final CompletableFuture<LogicalSlot> slotFuture = CompletableFuture.completedFuture(slot); final CountDownLatch slotRequestLatch = new CountDownLatch(1); final TestingSlotProvider slotProvider = new TestingSlotProvider(slotRequestId -> { slotRequestLatch.countDown(); return slotFuture; }); final ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph( new JobID(), slotProvider, new NoRestartStrategy(), jobVertex); final Execution execution = executionGraph.getJobVertex(jobVertex.getID()).getTaskVertices()[0].getCurrentExecutionAttempt(); executionGraph.start(testMainThreadUtil.getMainThreadExecutor()); testMainThreadUtil.execute(executionGraph::scheduleForExecution); // wait until the slot has been requested slotRequestLatch.await(); testMainThreadUtil.execute(() -> { assertThat(execution.getAssignedResource(), is(sameInstance(slot))); slot.release(new FlinkException("Test exception")); assertThat(execution.getReleaseFuture().isDone(), is(true)); }); }
Example #27
Source File: SlotPoolSlotSharingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSingleQueuedSharedSlotScheduling() throws Exception { final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>(); final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway(); testingResourceManagerGateway.setRequestSlotConsumer( (SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId())); LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation(); final SlotPoolImpl slotPool = slotPoolResource.getSlotPool(); slotPool.registerTaskManager(taskManagerLocation.getResourceID()); SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId(); final SlotProvider slotProvider = slotPoolResource.getSlotProvider(); CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot( new ScheduledUnit( new JobVertexID(), slotSharingGroupId, null), true, SlotProfile.noRequirements(), TestingUtils.infiniteTime()); assertFalse(logicalSlotFuture.isDone()); final AllocationID allocationId = allocationIdFuture.get(); boolean booleanCompletableFuture = slotPool.offerSlot( taskManagerLocation, new SimpleAckingTaskManagerGateway(), new SlotOffer( allocationId, 0, ResourceProfile.UNKNOWN)); assertTrue(booleanCompletableFuture); final LogicalSlot logicalSlot = logicalSlotFuture.get(); assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId()); }
Example #28
Source File: ExecutionGraphSchedulingTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that a partially completed eager scheduling operation fails if a * completed slot is released. See FLINK-9099. */ @Test public void testSlotReleasingFailsSchedulingOperation() throws Exception { final int parallelism = 2; final JobVertex jobVertex = new JobVertex("Testing job vertex"); jobVertex.setInvokableClass(NoOpInvokable.class); jobVertex.setParallelism(parallelism); final JobGraph jobGraph = new JobGraph(jobVertex); jobGraph.setAllowQueuedScheduling(true); jobGraph.setScheduleMode(ScheduleMode.EAGER); final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism); final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId()); slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot)); final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>(); slotProvider.addSlot(jobVertex.getID(), 1, slotFuture); final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider); executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread()); executionGraph.scheduleForExecution(); assertThat(executionGraph.getState(), is(JobStatus.RUNNING)); final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID()); final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices(); assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED)); assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED)); // fail the single allocated slot --> this should fail the scheduling operation slot.releaseSlot(new FlinkException("Test failure")); assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED)); }
Example #29
Source File: DefaultSchedulerTest.java From flink with Apache License 2.0 | 5 votes |
/** * Since checkpoint is triggered asynchronously, we need to figure out when checkpoint is really * triggered. * Note that this should be invoked before scheduler initialized. * * @return the latch representing checkpoint is really triggered */ private CountDownLatch getCheckpointTriggeredLatch() { final CountDownLatch checkpointTriggeredLatch = new CountDownLatch(1); final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway(); testExecutionSlotAllocator.getLogicalSlotBuilder().setTaskManagerGateway(taskManagerGateway); taskManagerGateway.setCheckpointConsumer( (executionAttemptID, jobId, checkpointId, timestamp, checkpointOptions, advanceToEndOfEventTime) -> { checkpointTriggeredLatch.countDown(); }); return checkpointTriggeredLatch; }
Example #30
Source File: ExecutionPartitionLifecycleTest.java From flink with Apache License 2.0 | 5 votes |
private void testPartitionTrackingForStateTransition(final Consumer<Execution> stateTransition, final PartitionReleaseResult partitionReleaseResult) throws Exception { CompletableFuture<Tuple2<ResourceID, ResultPartitionDeploymentDescriptor>> partitionStartTrackingFuture = new CompletableFuture<>(); CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingFuture = new CompletableFuture<>(); CompletableFuture<Collection<ResultPartitionID>> partitionStopTrackingAndReleaseFuture = new CompletableFuture<>(); final TestingPartitionTracker partitionTracker = new TestingPartitionTracker(); partitionTracker.setStartTrackingPartitionsConsumer( (resourceID, resultPartitionDeploymentDescriptor) -> partitionStartTrackingFuture.complete(Tuple2.of(resourceID, resultPartitionDeploymentDescriptor)) ); partitionTracker.setStopTrackingPartitionsConsumer(partitionStopTrackingFuture::complete); partitionTracker.setStopTrackingAndReleasePartitionsConsumer(partitionStopTrackingAndReleaseFuture::complete); setupExecutionGraphAndStartRunningJob(ResultPartitionType.BLOCKING, partitionTracker, new SimpleAckingTaskManagerGateway(), NettyShuffleMaster.INSTANCE); Tuple2<ResourceID, ResultPartitionDeploymentDescriptor> startTrackingCall = partitionStartTrackingFuture.get(); assertThat(startTrackingCall.f0, equalTo(taskExecutorResourceId)); assertThat(startTrackingCall.f1, equalTo(descriptor)); stateTransition.accept(execution); switch (partitionReleaseResult) { case NONE: assertFalse(partitionStopTrackingFuture.isDone()); assertFalse(partitionStopTrackingAndReleaseFuture.isDone()); break; case STOP_TRACKING: assertTrue(partitionStopTrackingFuture.isDone()); assertFalse(partitionStopTrackingAndReleaseFuture.isDone()); final Collection<ResultPartitionID> stopTrackingCall = partitionStopTrackingFuture.get(); assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingCall); break; case STOP_TRACKING_AND_RELEASE: assertFalse(partitionStopTrackingFuture.isDone()); assertTrue(partitionStopTrackingAndReleaseFuture.isDone()); final Collection<ResultPartitionID> stopTrackingAndReleaseCall = partitionStopTrackingAndReleaseFuture.get(); assertEquals(Collections.singletonList(descriptor.getShuffleDescriptor().getResultPartitionID()), stopTrackingAndReleaseCall); break; } }