org.apache.flink.runtime.testingUtils.TestingUtils Java Examples

The following examples show how to use org.apache.flink.runtime.testingUtils.TestingUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FutureUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that all scheduled tasks are canceled if the retry future is being cancelled.
 */
@Test
public void testRetryWithDelayCancellation() {
	final ManuallyTriggeredScheduledExecutor scheduledExecutor = new ManuallyTriggeredScheduledExecutor();

	CompletableFuture<?> retryFuture = FutureUtils.retryWithDelay(
		() -> FutureUtils.completedExceptionally(new FlinkException("Test exception")),
		1,
		TestingUtils.infiniteTime(),
		scheduledExecutor);

	assertFalse(retryFuture.isDone());

	final Collection<ScheduledFuture<?>> scheduledTasks = scheduledExecutor.getScheduledTasks();

	assertFalse(scheduledTasks.isEmpty());

	final ScheduledFuture<?> scheduledFuture = scheduledTasks.iterator().next();

	assertFalse(scheduledFuture.isDone());

	retryFuture.cancel(false);

	assertTrue(retryFuture.isCancelled());
	assertTrue(scheduledFuture.isCancelled());
}
 
Example #2
Source File: ResourceManagerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that we can retrieve the correct {@link TaskManagerInfo} from the {@link ResourceManager}.
 */
@Test
public void testRequestTaskManagerInfo() throws Exception {
	final ResourceID taskManagerId = ResourceID.generate();
	final TaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().setAddress(UUID.randomUUID().toString()).createTestingTaskExecutorGateway();
	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	resourceManager = createAndStartResourceManager(heartbeatServices);
	final ResourceManagerGateway resourceManagerGateway = resourceManager.getSelfGateway(ResourceManagerGateway.class);

	registerTaskExecutor(resourceManagerGateway, taskManagerId, taskExecutorGateway.getAddress());

	CompletableFuture<TaskManagerInfo> taskManagerInfoFuture = resourceManagerGateway.requestTaskManagerInfo(
		taskManagerId,
		TestingUtils.TIMEOUT());

	TaskManagerInfo taskManagerInfo = taskManagerInfoFuture.get();

	assertEquals(taskManagerId, taskManagerInfo.getResourceId());
	assertEquals(hardwareDescription, taskManagerInfo.getHardwareDescription());
	assertEquals(taskExecutorGateway.getAddress(), taskManagerInfo.getAddress());
	assertEquals(dataPort, taskManagerInfo.getDataPort());
	assertEquals(0, taskManagerInfo.getNumberSlots());
	assertEquals(0, taskManagerInfo.getNumberAvailableSlots());
}
 
Example #3
Source File: FutureUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSupplyAsyncFailure() throws Exception {
	final String exceptionMessage = "Test exception";
	final FlinkException testException = new FlinkException(exceptionMessage);
	final CompletableFuture<Object> future = FutureUtils.supplyAsync(
		() -> {
			throw testException;
		},
		TestingUtils.defaultExecutor());

	try {
		future.get();
		fail("Expected an exception.");
	} catch (ExecutionException e) {
		assertThat(ExceptionUtils.findThrowableWithMessage(e, exceptionMessage).isPresent(), is(true));
	}
}
 
Example #4
Source File: AggregatingMetricsHandlerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
 
Example #5
Source File: AggregatingMetricsHandlerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	MetricFetcher fetcher = new MetricFetcherImpl<RestfulGateway>(
		mock(GatewayRetriever.class),
		mock(MetricQueryServiceRetriever.class),
		Executors.directExecutor(),
		TestingUtils.TIMEOUT(),
		MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL.defaultValue());
	store = fetcher.getMetricStore();

	Collection<MetricDump> metricDumps = getMetricDumps();
	for (MetricDump dump : metricDumps) {
		store.add(dump);
	}

	handler = getHandler(
		LEADER_RETRIEVER,
		TIMEOUT,
		TEST_HEADERS,
		EXECUTOR,
		fetcher);
	pathParameters = getPathParameters();
}
 
Example #6
Source File: AbstractOperatorRestoreTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
private void restoreJob(ClassLoader classLoader, ClusterClient<?> clusterClient, Deadline deadline, String savepointPath) throws Exception {
	JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE);
	jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState));

	assertNotNull("Job doesn't have a JobID.", jobToRestore.getJobID());

	clusterClient.submitJob(jobToRestore, classLoader);

	CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay(
		() -> clusterClient.getJobStatus(jobToRestore.getJobID()),
		Time.milliseconds(50),
		deadline,
		(jobStatus) -> jobStatus == JobStatus.FINISHED,
		TestingUtils.defaultScheduledExecutor());
	assertEquals(
		JobStatus.FINISHED,
		jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
}
 
Example #7
Source File: FutureUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a retry future is failed after all retries have been consumed.
 */
@Test(expected = FutureUtils.RetryException.class)
public void testRetryFailure() throws Throwable {
	final int retries = 3;

	CompletableFuture<?> retryFuture = FutureUtils.retry(
		() -> FutureUtils.completedExceptionally(new FlinkException("Test exception")),
		retries,
		TestingUtils.defaultExecutor());

	try {
		retryFuture.get();
	} catch (ExecutionException ee) {
		throw ExceptionUtils.stripExecutionException(ee);
	}
}
 
Example #8
Source File: FutureUtilsTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a scheduled task is canceled if the scheduled future is being cancelled.
 */
@Test
public void testScheduleWithDelayCancellation() {
	final ManuallyTriggeredScheduledExecutor scheduledExecutor = new ManuallyTriggeredScheduledExecutor();

	final Runnable noOpRunnable = () -> {};
	CompletableFuture<Void> completableFuture = FutureUtils.scheduleWithDelay(
		noOpRunnable,
		TestingUtils.infiniteTime(),
		scheduledExecutor);

	final ScheduledFuture<?> scheduledFuture = scheduledExecutor
		.getScheduledTasks()
		.iterator()
		.next();

	completableFuture.cancel(false);

	assertTrue(completableFuture.isCancelled());
	assertTrue(scheduledFuture.isCancelled());
}
 
Example #9
Source File: PipelinedFailoverRegionBuildingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 
Example #10
Source File: ExecutionGraphRestartTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private Tuple2<ExecutionGraph, Instance> createExecutionGraph(RestartStrategy restartStrategy) throws Exception {
	Instance instance = ExecutionGraphTestUtils.getInstance(
		new ActorTaskManagerGateway(
			new SimpleActorGateway(TestingUtils.directExecutionContext())),
		NUM_TASKS);

	Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
	scheduler.newInstanceAvailable(instance);

	ExecutionGraph eg = createSimpleExecutionGraph(restartStrategy, scheduler);

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();
	assertEquals(JobStatus.RUNNING, eg.getState());
	return new Tuple2<>(eg, instance);
}
 
Example #11
Source File: AbstractTaskManagerFileHandlerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private AbstractTaskManagerFileHandlerTest.TestTaskManagerFileHandler createTestTaskManagerFileHandler(
	Time cacheEntryDuration,
	Queue<CompletableFuture<TransientBlobKey>> requestFileUploads,
	ResourceID expectedTaskManagerId) {
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();

	return new TestTaskManagerFileHandler(
		() -> CompletableFuture.completedFuture(null),
		TestingUtils.infiniteTime(),
		Collections.emptyMap(),
		new TestUntypedMessageHeaders(),
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		cacheEntryDuration,
		requestFileUploads,
		expectedTaskManagerId);
}
 
Example #12
Source File: ScheduleWithCoLocationHintTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void scheduleWithReleaseNoResource() throws Exception {
	JobVertexID jid1 = new JobVertexID();
	JobVertexID jid2 = new JobVertexID();
	JobVertexID jid3 = new JobVertexID();

	testingSlotProvider.addTaskManager(1);
	testingSlotProvider.addTaskManager(1);

	assertEquals(2, testingSlotProvider.getNumberOfAvailableSlots());

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	CoLocationConstraint c1 = new CoLocationConstraint(new CoLocationGroup());

	LogicalSlot s1 = testingSlotProvider.allocateSlot(
			new ScheduledUnit(getTestVertex(jid1, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();
	s1.releaseSlot();

	testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 0, 1, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();
	testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid2, 1, 2, null)), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();

	try {
		testingSlotProvider.allocateSlot(new ScheduledUnit(getTestVertex(jid3, 0, 1, sharingGroup), sharingGroup.getSlotSharingGroupId(), c1), false, SlotProfile.noRequirements(), TestingUtils.infiniteTime()).get();
		fail("Scheduled even though no resource was available.");
	} catch (ExecutionException e) {
		assertTrue(e.getCause() instanceof NoResourceAvailableException);
	}

	assertEquals(0, testingSlotProvider.getNumberOfLocalizedAssignments());
	assertEquals(0, testingSlotProvider.getNumberOfNonLocalizedAssignments());
	assertEquals(3, testingSlotProvider.getNumberOfUnconstrainedAssignments());
}
 
Example #13
Source File: GlobalModVersionTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private ExecutionGraph createSampleGraph(FailoverStrategy failoverStrategy) throws Exception {

		final JobID jid = new JobID();
		final int parallelism = new Random().nextInt(10) + 1;

		final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jid, parallelism);

		// build a simple execution graph with on job vertex, parallelism 2
		final ExecutionGraph graph = new ExecutionGraph(
			new DummyJobInformation(
				jid,
				"test job"),
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			Time.seconds(10),
			new InfiniteDelayRestartStrategy(),
			new CustomStrategy(failoverStrategy),
			slotProvider);

		graph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

		JobVertex jv = new JobVertex("test vertex");
		jv.setInvokableClass(NoOpInvokable.class);
		jv.setParallelism(parallelism);

		JobGraph jg = new JobGraph(jid, "testjob", jv);
		graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

		return graph;
	}
 
Example #14
Source File: TaskManagerReleaseInSlotManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that idle task managers time out after the configured timeout. A timed out task manager
 * will be removed from the slot manager and the resource manager will be notified about the
 * timeout, if it can be released.
 */
@Test
public void testTaskManagerTimeout() throws Exception {
	Executor executor = TestingUtils.defaultExecutor();
	canBeReleasedFuture.set(CompletableFuture.completedFuture(true));
	try (SlotManager slotManager = SlotManagerBuilder
		.newBuilder()
		.setTaskManagerTimeout(Time.milliseconds(10L))
		.build()) {

		slotManager.start(resourceManagerId, executor, resourceManagerActions);
		executor.execute(() -> slotManager.registerTaskManager(taskManagerConnection, slotReport));
		assertThat(releaseFuture.get(), is(equalTo(taskManagerConnection.getInstanceID())));
	}
}
 
Example #15
Source File: HeartbeatManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that regular heartbeat signal triggers the right callback functions in the
 * {@link HeartbeatListener}.
 */
@Test
public void testRegularHeartbeat() throws InterruptedException {
	final long heartbeatTimeout = 1000L;
	ResourceID ownResourceID = new ResourceID("foobar");
	ResourceID targetResourceID = new ResourceID("barfoo");
	final int outputPayload = 42;
	final ArrayBlockingQueue<String> reportedPayloads = new ArrayBlockingQueue<>(2);
	final TestingHeartbeatListener<String, Integer> heartbeatListener = new TestingHeartbeatListenerBuilder<String, Integer>()
		.setReportPayloadConsumer((ignored, payload) -> reportedPayloads.offer(payload))
		.setRetrievePayloadFunction((ignored) -> outputPayload)
		.createNewTestingHeartbeatListener();

	HeartbeatManagerImpl<String, Integer> heartbeatManager = new HeartbeatManagerImpl<>(
		heartbeatTimeout,
		ownResourceID,
		heartbeatListener,
		TestingUtils.defaultScheduledExecutor(),
		LOG);

	final ArrayBlockingQueue<Integer> reportedPayloadsHeartbeatTarget = new ArrayBlockingQueue<>(2);
	final TestingHeartbeatTarget<Integer> heartbeatTarget = new TestingHeartbeatTargetBuilder<Integer>()
		.setReceiveHeartbeatConsumer((ignoredA, payload) -> reportedPayloadsHeartbeatTarget.offer(payload))
		.createTestingHeartbeatTarget();

	heartbeatManager.monitorTarget(targetResourceID, heartbeatTarget);

	final String inputPayload1 = "foobar";
	heartbeatManager.requestHeartbeat(targetResourceID, inputPayload1);

	assertThat(reportedPayloads.take(), is(inputPayload1));
	assertThat(reportedPayloadsHeartbeatTarget.take(), is(outputPayload));

	final String inputPayload2 = "barfoo";
	heartbeatManager.receiveHeartbeat(targetResourceID, inputPayload2);
	assertThat(reportedPayloads.take(), is(inputPayload2));
}
 
Example #16
Source File: ExecutionVertexLocalityTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 
Example #17
Source File: FutureUtilsTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the delay is respected between subsequent retries of a retry future with retry delay.
 */
@Test
public void testRetryWithDelay() throws Exception {
	final int retries = 4;
	final Time delay = Time.milliseconds(5L);
	final AtomicInteger countDown = new AtomicInteger(retries);

	long start = System.currentTimeMillis();

	CompletableFuture<Boolean> retryFuture = FutureUtils.retryWithDelay(
		() -> {
			if (countDown.getAndDecrement() == 0) {
				return CompletableFuture.completedFuture(true);
			} else {
				return FutureUtils.completedExceptionally(new FlinkException("Test exception."));
			}
		},
		retries,
		delay,
		TestingUtils.defaultScheduledExecutor());

	Boolean result = retryFuture.get();

	long completionTime = System.currentTimeMillis() - start;

	assertTrue(result);
	assertTrue("The completion time should be at least rertries times delay between retries.", completionTime >= retries * delay.toMilliseconds());
}
 
Example #18
Source File: SlotPoolImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
private TestingSlotPoolImpl createSlotPoolImpl(ManualClock clock) {
	return new TestingSlotPoolImpl(
		jobId,
		clock,
		TestingUtils.infiniteTime(),
		timeout,
		TestingUtils.infiniteTime());
}
 
Example #19
Source File: ExecutionGraphRescalingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER,
			NettyShuffleMaster.INSTANCE,
			NoOpJobMasterPartitionTracker.INSTANCE);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 
Example #20
Source File: JobExceptionsHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetJobExceptionsInfo() throws HandlerRequestException {
	final JobExceptionsHandler jobExceptionsHandler = new JobExceptionsHandler(
		() -> null,
		TestingUtils.TIMEOUT(),
		Collections.emptyMap(),
		JobExceptionsHeaders.getInstance(),
		new DefaultExecutionGraphCache(TestingUtils.TIMEOUT(), TestingUtils.TIMEOUT()),
		TestingUtils.defaultExecutor());
	final int numExceptions = 20;
	final AccessExecutionGraph archivedExecutionGraph = createAccessExecutionGraph(numExceptions);
	checkExceptionLimit(jobExceptionsHandler, archivedExecutionGraph, numExceptions, 10);
	checkExceptionLimit(jobExceptionsHandler, archivedExecutionGraph, numExceptions, numExceptions);
	checkExceptionLimit(jobExceptionsHandler, archivedExecutionGraph, numExceptions, 30);
}
 
Example #21
Source File: ProcessFailureCancelingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException {
	FutureUtils.retrySuccessfulWithDelay(
		() -> dispatcherGateway.requestClusterOverview(timeout),
		Time.milliseconds(50L),
		Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())),
		clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= 1 &&
			clusterOverview.getNumSlotsAvailable() == 0 &&
			clusterOverview.getNumSlotsTotal() == 2,
		TestingUtils.defaultScheduledExecutor())
		.get();
}
 
Example #22
Source File: JobSubmitHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testRejectionOnCountMismatch() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}
	final Path countExceedingFile = TEMPORARY_FOLDER.newFile().toPath();

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Arrays.asList(jobGraphFile.toFile(), countExceedingFile.toFile())), mockGateway)
			.get();
	} catch (Exception e) {
		ExceptionUtils.findThrowable(e, candidate -> candidate instanceof RestHandlerException && candidate.getMessage().contains("count"));
	}
}
 
Example #23
Source File: ResourceManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TestingResourceManager createAndStartResourceManager(HeartbeatServices heartbeatServices) throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
		resourceManagerResourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpMetricRegistry.INSTANCE,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerId = ResourceManagerId.generate();
	resourceManagerLeaderElectionService.isLeader(resourceManagerId.toUUID()).get();

	return resourceManager;
}
 
Example #24
Source File: SlotManagerBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
private SlotManagerBuilder() {
	this.slotMatchingStrategy = AnyMatchingSlotMatchingStrategy.INSTANCE;
	this.scheduledExecutor = TestingUtils.defaultScheduledExecutor();
	this.taskManagerRequestTimeout = TestingUtils.infiniteTime();
	this.slotRequestTimeout = TestingUtils.infiniteTime();
	this.taskManagerTimeout = TestingUtils.infiniteTime();
	this.waitResultConsumedBeforeRelease = true;
	this.defaultWorkerResourceSpec = WorkerResourceSpec.ZERO;
	this.numSlotsPerWorker = 1;
	this.slotManagerMetricGroup = UnregisteredMetricGroups.createUnregisteredSlotManagerMetricGroup();
	this.maxSlotNum = ResourceManagerOptions.MAX_SLOT_NUM.defaultValue();
}
 
Example #25
Source File: SlotManagerBuilder.java    From flink with Apache License 2.0 5 votes vote down vote up
private SlotManagerBuilder() {
	this.scheduledExecutor = TestingUtils.defaultScheduledExecutor();
	this.taskManagerRequestTimeout = TestingUtils.infiniteTime();
	this.slotRequestTimeout = TestingUtils.infiniteTime();
	this.taskManagerTimeout = TestingUtils.infiniteTime();
	this.waitResultConsumedBeforeRelease = true;
}
 
Example #26
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that returned slot futures are failed if the allocation request is failed.
 */
@Test
public void testFailingQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			new SlotSharingGroupId(),
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId = allocationIdFuture.get();

	// this should fail the returned logical slot future
	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.failAllocation(allocationId, new FlinkException("Testing Exception"));

	try {
		logicalSlotFuture.get();
		fail("The slot future should have failed.");
	} catch (ExecutionException ee) {
		assertTrue(ExceptionUtils.findThrowable(ee, FlinkException.class).isPresent());
	}
}
 
Example #27
Source File: JobManagerRunnerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Nonnull
private JobManagerRunner createJobManagerRunner(JobMasterServiceFactory jobMasterServiceFactory, LibraryCacheManager libraryCacheManager) throws Exception{
	return new JobManagerRunner(
		jobGraph,
		jobMasterServiceFactory,
		haServices,
		libraryCacheManager,
		TestingUtils.defaultExecutor(),
		fatalErrorHandler);
}
 
Example #28
Source File: ExecutionGraphRescalingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Verifies that building an {@link ExecutionGraph} from a {@link JobGraph} with
 * parallelism higher than the maximum parallelism fails.
 */
@Test
public void testExecutionGraphConstructionFailsRescaleDopExceedMaxParallelism() throws Exception {

	final Configuration config = new Configuration();

	final int initialParallelism = 1;
	final int maxParallelism = 10;
	final JobVertex[] jobVertices = createVerticesForSimpleBipartiteJobGraph(initialParallelism,  maxParallelism);
	final JobGraph jobGraph = new JobGraph(jobVertices);

	for (JobVertex jv : jobVertices) {
		jv.setParallelism(maxParallelism + 1);
	}

	try {
		// this should fail since we set the parallelism to maxParallelism + 1
		ExecutionGraphBuilder.buildGraph(
			null,
			jobGraph,
			config,
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			new TestingSlotProvider(ignore -> new CompletableFuture<>()),
			Thread.currentThread().getContextClassLoader(),
			new StandaloneCheckpointRecoveryFactory(),
			AkkaUtils.getDefaultTimeout(),
			new NoRestartStrategy(),
			new UnregisteredMetricsGroup(),
			VoidBlobWriter.getInstance(),
			AkkaUtils.getDefaultTimeout(),
			TEST_LOGGER);

		fail("Building the ExecutionGraph with a parallelism higher than the max parallelism should fail.");
	} catch (JobException e) {
		// expected, ignore
	}
}
 
Example #29
Source File: ResourceManagerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private TestingResourceManager createAndStartResourceManager(HeartbeatServices heartbeatServices) throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		ResourceManager.RESOURCE_MANAGER_NAME + UUID.randomUUID(),
		resourceManagerResourceId,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		NoOpMetricRegistry.INSTANCE,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerId = ResourceManagerId.generate();
	resourceManagerLeaderElectionService.isLeader(resourceManagerId.toUUID()).get();

	return resourceManager;
}
 
Example #30
Source File: ResourceManagerPartitionLifecycleTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private ResourceManagerGateway createAndStartResourceManager() throws Exception {
	final SlotManager slotManager = SlotManagerBuilder.newBuilder()
		.setScheduledExecutor(rpcService.getScheduledExecutor())
		.build();
	final JobLeaderIdService jobLeaderIdService = new JobLeaderIdService(
		highAvailabilityServices,
		rpcService.getScheduledExecutor(),
		TestingUtils.infiniteTime());

	final TestingResourceManager resourceManager = new TestingResourceManager(
		rpcService,
		ResourceID.generate(),
		highAvailabilityServices,
		new HeartbeatServices(100000L, 1000000L),
		slotManager,
		ResourceManagerPartitionTrackerImpl::new,
		jobLeaderIdService,
		testingFatalErrorHandler,
		UnregisteredMetricGroups.createUnregisteredResourceManagerMetricGroup());

	resourceManager.start();

	// first make the ResourceManager the leader
	resourceManagerLeaderElectionService.isLeader(ResourceManagerId.generate().toUUID()).get();

	this.resourceManager = resourceManager;

	return resourceManager.getSelfGateway(ResourceManagerGateway.class);
}