org.apache.flink.util.FlinkException Java Examples

@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
	final PackagedProgram packagedProgram = createPackagedProgram();
	final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
	try {
		final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(
			packagedProgram,
			configuration,
			defaultParallelism,
			jobId);
		jobGraph.setAllowQueuedScheduling(true);
		jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

		return jobGraph;
	} catch (Exception e) {
		throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
	}
}
 

private CompletableFuture<JobGraph> uploadJobGraphFiles(
		DispatcherGateway gateway,
		CompletableFuture<JobGraph> jobGraphFuture,
		Collection<Path> jarFiles,
		Collection<Tuple2<String, Path>> artifacts,
		Configuration configuration) {
	CompletableFuture<Integer> blobServerPortFuture = gateway.getBlobServerPort(timeout);

	return jobGraphFuture.thenCombine(blobServerPortFuture, (JobGraph jobGraph, Integer blobServerPort) -> {
		final InetSocketAddress address = new InetSocketAddress(gateway.getHostname(), blobServerPort);
		try {
			ClientUtils.uploadJobGraphFiles(jobGraph, jarFiles, artifacts, () -> new BlobClient(address, configuration));
		} catch (FlinkException e) {
			throw new CompletionException(new RestHandlerException(
				"Could not upload job files.",
				HttpResponseStatus.INTERNAL_SERVER_ERROR,
				e));
		}
		return jobGraph;
	});
}
 

/**
 * Tests the triggering and exceptional completion of an asynchronous operation.
 */
@Test
public void testOperationFailure() throws Exception {
	final FlinkException testException = new FlinkException("Test exception");
	final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder()
		.setTriggerSavepointFunction((JobID jobId, String directory) -> FutureUtils.completedExceptionally(testException))
		.build();

	// trigger the operation
	final TriggerId triggerId = testingTriggerHandler.handleRequest(
		triggerOperationRequest(),
		testingRestfulGateway).get().getTriggerId();

	AsynchronousOperationResult<OperationResult> operationResult = testingStatusHandler.handleRequest(
		statusOperationRequest(triggerId),
		testingRestfulGateway).get();

	assertThat(operationResult.queueStatus().getId(), is(QueueStatus.completed().getId()));

	final OperationResult resource = operationResult.resource();
	assertThat(resource.throwable, is(testException));
}
 

private void closeJobManagerConnectionIfNoAllocatedResources(JobID jobId) {
	// check whether we still have allocated slots for the same job
	if (taskSlotTable.getAllocationIdsPerJob(jobId).isEmpty() && !partitionTable.hasTrackedPartitions(jobId)) {
		// we can remove the job from the job leader service
		try {
			jobLeaderService.removeJob(jobId);
		} catch (Exception e) {
			log.info("Could not remove job {} from JobLeaderService.", jobId, e);
		}

		closeJobManagerConnection(
			jobId,
			new FlinkException("TaskExecutor " + getAddress() +
				" has no more allocated slots for job " + jobId + '.'));
	}
}
 

/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 

@Test
public void testInvokableInstantiationFailed() throws Exception {
	final QueuedNoOpTaskManagerActions taskManagerActions = new QueuedNoOpTaskManagerActions();
	final Task task = new TaskBuilder()
		.setTaskManagerActions(taskManagerActions)
		.setInvokable(InvokableNonInstantiable.class)
		.build();

	// should fail
	task.run();

	// verify final state
	assertEquals(ExecutionState.FAILED, task.getExecutionState());
	assertTrue(task.isCanceledOrFailed());
	assertTrue(task.getFailureCause().getMessage().contains("instantiate"));

	taskManagerActions.validateListenerMessage(
		ExecutionState.FAILED, task, new FlinkException("Could not instantiate the task's invokable class."));
}
 

@Override
public CompletableFuture<Acknowledge> triggerMainThreadExecutorComputation(Time timeout) {
	return CompletableFuture.supplyAsync(
		() -> {
			try {
				computationLatch.await();
			} catch (InterruptedException e) {
				throw new CompletionException(new FlinkException("Waiting on latch failed.", e));
			}

			return value;
		},
		getRpcService().getExecutor())
	.thenApplyAsync(
		(String v) -> Acknowledge.get(),
		getMainThreadExecutor());
}
 

private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException {
	@SuppressWarnings("unchecked")
	final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook;
	final String id = hook.getIdentifier();

	try {
		final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer();
		if (deserializer == null) {
			throw new FlinkException("null serializer for state of hook " + hook.getIdentifier());
		}

		return deserializer.deserialize(state.version(), state.bytes());
	}
	catch (Throwable t) {
		throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t);
	}
}
 

private void startJobMasterServices() throws Exception {
	startHeartbeatServices();

	// start the slot pool make sure the slot pool now accepts messages for this leader
	slotPool.start(getFencingToken(), getAddress(), getMainThreadExecutor());
	scheduler.start(getMainThreadExecutor());

	//TODO: Remove once the ZooKeeperLeaderRetrieval returns the stored address upon start
	// try to reconnect to previously known leader
	reconnectToResourceManager(new FlinkException("Starting JobMaster component."));

	// job is ready to go, try to establish connection with resource manager
	//   - activate leader retrieval for the resource manager
	//   - on notification of the leader, the connection will be established and
	//     the slot pool will start requesting slots
	resourceManagerLeaderRetriever.start(new ResourceManagerLeaderListener());
}
 

private void initializeLocation(ExecutionVertex vertex, TaskManagerLocation location) throws Exception {
	// we need a bit of reflection magic to initialize the location without going through
	// scheduling paths. we choose to do that, rather than the alternatives:
	//  - mocking the scheduler created fragile tests that break whenever the scheduler is adjusted
	//  - exposing test methods in the ExecutionVertex leads to undesirable setters 

	SlotContext slot = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		mock(TaskManagerGateway.class));

	SimpleSlot simpleSlot = new SimpleSlot(slot, mock(SlotOwner.class), 0);

	if (!vertex.getCurrentExecutionAttempt().tryAssignResource(simpleSlot)) {
		throw new FlinkException("Could not assign resource.");
	}
}
 

/**
 * Tests the triggering and exceptional completion of an asynchronous operation.
 */
@Test
public void testOperationFailure() throws Exception {
	final FlinkException testException = new FlinkException("Test exception");
	final TestingRestfulGateway testingRestfulGateway = new TestingRestfulGateway.Builder()
		.setTriggerSavepointFunction((JobID jobId, String directory) -> FutureUtils.completedExceptionally(testException))
		.build();

	// trigger the operation
	final TriggerId triggerId = testingTriggerHandler.handleRequest(
		triggerOperationRequest(),
		testingRestfulGateway).get().getTriggerId();

	AsynchronousOperationResult<OperationResult> operationResult = testingStatusHandler.handleRequest(
		statusOperationRequest(triggerId),
		testingRestfulGateway).get();

	assertThat(operationResult.queueStatus().getId(), is(QueueStatus.completed().getId()));

	final OperationResult resource = operationResult.resource();
	assertThat(resource.throwable, is(testException));
}
 

/**
 * Tests that the {@link Dispatcher} terminates if it cannot recover jobs from
 * the {@link SubmittedJobGraphStore}. See FLINK-8943.
 */
@Test
public void testFatalErrorAfterJobRecoveryFailure() throws Exception {
	final FlinkException testException = new FlinkException("Test exception");

	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	dispatcher.waitUntilStarted();

	final SubmittedJobGraph submittedJobGraph = new SubmittedJobGraph(jobGraph);
	submittedJobGraphStore.putJobGraph(submittedJobGraph);

	submittedJobGraphStore.setRecoverJobGraphFunction(
		(JobID jobId, Map<JobID, SubmittedJobGraph> submittedJobs) -> {
			throw testException;
		});

	electDispatcher();

	// we expect that a fatal error occurred
	final Throwable error = fatalErrorHandler.getErrorFuture().get(TIMEOUT.toMilliseconds(), TimeUnit.MILLISECONDS);

	assertThat(ExceptionUtils.findThrowableWithMessage(error, testException.getMessage()).isPresent(), is(true));

	fatalErrorHandler.clearError();
}
 

/**
 * Tests that a fatal error is reported if the job recovery fails.
 */
@Test
public void testFailingRecoveryIsAFatalError() throws Exception {
	final String exceptionMessage = "Job recovery test failure.";
	final Supplier<Exception> exceptionSupplier = () -> new FlinkException(exceptionMessage);
	final TestingHighAvailabilityServices haServices = new TestingHighAvailabilityServicesBuilder()
		.setSubmittedJobGraphStore(new FailingSubmittedJobGraphStore(exceptionSupplier))
		.build();

	final HATestingDispatcher dispatcher = createDispatcher(haServices);
	dispatcher.start();

	final Throwable failure = testingFatalErrorHandler.getErrorFuture().get();

	assertThat(ExceptionUtils.findThrowableWithMessage(failure, exceptionMessage).isPresent(), is(true));

	testingFatalErrorHandler.clearError();
}
 

private static CompletedCheckpoint retrieveCompletedCheckpoint(Tuple2<RetrievableStateHandle<CompletedCheckpoint>, String> stateHandlePath) throws FlinkException {
	long checkpointId = pathToCheckpointId(stateHandlePath.f1);

	LOG.info("Trying to retrieve checkpoint {}.", checkpointId);

	try {
		return stateHandlePath.f0.retrieveState();
	} catch (ClassNotFoundException cnfe) {
		throw new FlinkException("Could not retrieve checkpoint " + checkpointId + " from state handle under " +
			stateHandlePath.f1 + ". This indicates that you are trying to recover from state written by an " +
			"older Flink version which is not compatible. Try cleaning the state handle store.", cnfe);
	} catch (IOException ioe) {
		throw new FlinkException("Could not retrieve checkpoint " + checkpointId + " from state handle under " +
			stateHandlePath.f1 + ". This indicates that the retrieved state handle is broken. Try cleaning the " +
			"state handle store.", ioe);
	}
}
 

@Test
public void testClosePartitionDiscovererWhenFetcherFails() throws Exception {
    final FlinkException failureCause = new FlinkException("Run fetcher failure.");

    // in this scenario, the partition discoverer will be concurrently accessed;
    // use the WakeupBeforeCloseTestingPartitionDiscoverer to verify that we always call
    // wakeup() before closing the discoverer
    final DummyPartitionDiscoverer testDiscoverer = new DummyPartitionDiscoverer();
    final PulsarFetcher<String> mock = mock(PulsarFetcher.class);
    doThrow(failureCause).when(mock).runFetchLoop();
    final DummyFlinkPulsarSource<String> source = new DummyFlinkPulsarSource<>(
            () -> mock,
            testDiscoverer,
            dummyProperties);

    testFailingSourceLifecycle(source, failureCause);
    assertTrue("partitionDiscoverer should be closed when consumer is closed", testDiscoverer.isClosed());
}
 

/**
 * Called from {@link org.apache.flink.runtime.instance.SharedSlot#releaseSlot(Throwable)}.
 * 
 * @param sharedSlot The slot to be released.
 */
void releaseSharedSlot(SharedSlot sharedSlot) {
	synchronized (lock) {
		if (sharedSlot.markCancelled()) {
			// we are releasing this slot
			
			if (sharedSlot.hasChildren()) {
				final FlinkException cause = new FlinkException("Releasing shared slot parent.");
				// by simply releasing all children, we should eventually release this slot.
				Set<Slot> children = sharedSlot.getSubSlots();
				while (children.size() > 0) {
					children.iterator().next().releaseSlot(cause);
				}
			}
			else {
				// if there are no children that trigger the release, we trigger it directly
				internalDisposeEmptySharedSlot(sharedSlot);
			}
		}
	}
}
 

/**
 * Loads the {@link WebMonitorExtension} which enables web submission.
 *
 * @param leaderRetriever to retrieve the leader
 * @param timeout for asynchronous requests
 * @param responseHeaders for the web submission handlers
 * @param localAddressFuture of the underlying REST server endpoint
 * @param uploadDir where the web submission handler store uploaded jars
 * @param executor to run asynchronous operations
 * @param configuration used to instantiate the web submission extension
 * @return Web submission extension
 * @throws FlinkException if the web submission extension could not be loaded
 */
public static WebMonitorExtension loadWebSubmissionExtension(
		GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		CompletableFuture<String> localAddressFuture,
		java.nio.file.Path uploadDir,
		Executor executor,
		Configuration configuration) throws FlinkException {

	if (isFlinkRuntimeWebInClassPath()) {
		try {
			final Constructor<?> webSubmissionExtensionConstructor = Class
				.forName("org.apache.flink.runtime.webmonitor.WebSubmissionExtension")
				.getConstructor(
					Configuration.class,
					GatewayRetriever.class,
					Map.class,
					CompletableFuture.class,
					java.nio.file.Path.class,
					Executor.class,
					Time.class);

			return (WebMonitorExtension) webSubmissionExtensionConstructor.newInstance(
				configuration,
				leaderRetriever,
				responseHeaders,
				localAddressFuture,
				uploadDir,
				executor,
				timeout);
		} catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException | IllegalAccessException e) {
			throw new FlinkException("Could not load web submission extension.", e);
		}
	} else {
		throw new FlinkException("The module flink-runtime-web could not be found in the class path. Please add " +
			"this jar in order to enable web based job submission.");
	}
}
 

@Override
protected CompletableFuture<TransientBlobKey> requestFileUpload(ResourceManagerGateway resourceManagerGateway, ResourceID taskManagerResourceId) {
	assertThat(taskManagerResourceId, is(equalTo(expectedTaskManagerId)));
	final CompletableFuture<TransientBlobKey> transientBlobKeyFuture = requestFileUploads.poll();

	if (transientBlobKeyFuture != null) {
		return transientBlobKeyFuture;
	} else {
		return FutureUtils.completedExceptionally(new FlinkException("Could not upload file."));
	}
}
 

@Override
public Map<String, OptionalFailure<Object>> getAccumulators(final JobID jobID, ClassLoader loader) throws Exception {
	final JobAccumulatorsHeaders accumulatorsHeaders = JobAccumulatorsHeaders.getInstance();
	final JobAccumulatorsMessageParameters accMsgParams = accumulatorsHeaders.getUnresolvedMessageParameters();
	accMsgParams.jobPathParameter.resolve(jobID);
	accMsgParams.includeSerializedAccumulatorsParameter.resolve(Collections.singletonList(true));

	CompletableFuture<JobAccumulatorsInfo> responseFuture = sendRequest(
		accumulatorsHeaders,
		accMsgParams);

	Map<String, OptionalFailure<Object>> result = Collections.emptyMap();

	try {
		result = responseFuture.thenApply((JobAccumulatorsInfo accumulatorsInfo) -> {
			try {
				return AccumulatorHelper.deserializeAccumulators(
					accumulatorsInfo.getSerializedUserAccumulators(),
					loader);
			} catch (Exception e) {
				throw new CompletionException(
					new FlinkException(
						String.format("Deserialization of accumulators for job %s failed.", jobID),
						e));
			}
		}).get(timeout.toMillis(), TimeUnit.MILLISECONDS);
	} catch (ExecutionException ee) {
		ExceptionUtils.rethrowException(ExceptionUtils.stripExecutionException(ee));
	}

	return result;
}
 

private void handleException(CompletableFuture<Void> leadershipOperation, String message) {
	leadershipOperation.whenComplete(
		(ignored, throwable) -> {
			if (throwable != null) {
				handleJobManagerRunnerError(new FlinkException(message, throwable));
			}
		});
}
 

/**
 * Tests that concurrent release operations only trigger the failing of the payload and
 * the return of the slot once.
 */
@Test
public void testConcurrentReleaseOperations() throws Exception {
	final CountingSlotOwner countingSlotOwner = new CountingSlotOwner();
	final CountingFailPayload countingFailPayload = new CountingFailPayload();
	final SingleLogicalSlot singleLogicalSlot = createSingleLogicalSlot(countingSlotOwner);

	singleLogicalSlot.tryAssignPayload(countingFailPayload);

	final ExecutorService executorService = Executors.newFixedThreadPool(4);

	try {
		final int numberConcurrentOperations = 10;
		final Collection<CompletableFuture<?>> releaseOperationFutures = new ArrayList<>(numberConcurrentOperations);

		for (int i = 0; i < numberConcurrentOperations; i++) {
			final CompletableFuture<Void> releaseOperationFuture = CompletableFuture.runAsync(
				() -> {
					try {
						singleLogicalSlot.releaseSlot(new FlinkException("Test exception")).get();
					} catch (InterruptedException | ExecutionException e) {
						ExceptionUtils.checkInterrupted(e);
						throw new CompletionException(e);
					}
				});

			releaseOperationFutures.add(releaseOperationFuture);
		}

		final FutureUtils.ConjunctFuture<Void> releaseOperationsFuture = FutureUtils.waitForAll(releaseOperationFutures);

		releaseOperationsFuture.get();

		assertThat(countingSlotOwner.getReleaseCount(), is(1));
		assertThat(countingFailPayload.getFailCount(), is(1));
	} finally {
		executorService.shutdownNow();
	}
}
 

@Override
public AbstractYarnClusterDescriptor createClusterDescriptor(CommandLine commandLine)
	throws FlinkException {
	AbstractYarnClusterDescriptor parent = super.createClusterDescriptor(commandLine);
	return new NonDeployingDetachedYarnClusterDescriptor(
			parent.getFlinkConfiguration(),
			(YarnConfiguration) parent.getYarnClient().getConfig(),
			configurationDirectory,
			parent.getYarnClient(),
			clusterClient);
}
 

private CompletableFuture<Void> revokeJobMasterLeadership() {
	log.info("JobManager for job {} ({}) was revoked leadership at {}.",
		jobGraph.getName(), jobGraph.getJobID(), getAddress());

	setNewLeaderGatewayFuture();

	return jobMasterService
		.suspend(new FlinkException("JobManager is no longer the leader."))
		.thenApply(FunctionUtils.nullFn());
}
 

@Test(expected = FlinkException.class)
public void testFailsWithNoTimestampsRight() throws Exception {
	try (TestHarness newTestHarness = createTestHarness(0L, true, 0L, true)) {

		newTestHarness.setup();
		newTestHarness.open();

		// note that the StreamRecord has no timestamp in constructor
		newTestHarness.processElement2(new StreamRecord<>(new TestElem(0, "rhs")));
	}
}
 

/**
 * Checks whether the given job has already been submitted or executed.
 *
 * @param jobId identifying the submitted job
 * @return true if the job has already been submitted (is running) or has been executed
 * @throws FlinkException if the job scheduling status cannot be retrieved
 */
private boolean isDuplicateJob(JobID jobId) throws FlinkException {
	final RunningJobsRegistry.JobSchedulingStatus jobSchedulingStatus;

	try {
		jobSchedulingStatus = runningJobsRegistry.getJobSchedulingStatus(jobId);
	} catch (IOException e) {
		throw new FlinkException(String.format("Failed to retrieve job scheduling status for job %s.", jobId), e);
	}

	return jobSchedulingStatus == RunningJobsRegistry.JobSchedulingStatus.DONE || jobManagerRunnerFutures.containsKey(jobId);
}
 

@Test
public void testTriggerSavepointFailure() throws Exception {
	replaceStdOutAndStdErr();

	JobID jobId = new JobID();

	String expectedTestException = "expectedTestException";
	Exception testException = new Exception(expectedTestException);

	final ClusterClient<String> clusterClient = createFailingClusterClient(testException);

	try {
		MockedCliFrontend frontend = new MockedCliFrontend(clusterClient);

		String[] parameters = { jobId.toString() };

		try {
			frontend.savepoint(parameters);

			fail("Savepoint should have failed.");
		} catch (FlinkException e) {
			assertTrue(ExceptionUtils.findThrowableWithMessage(e, expectedTestException).isPresent());
		}
	}
	finally {
		clusterClient.shutdown();
		restoreStdOutAndStdErr();
	}
}
 

@Override
protected CompletableFuture<RpcGateway> createGateway(CompletableFuture<Tuple2<String, UUID>> leaderFuture) {
	CompletableFuture<RpcGateway> result;

	if (retrievalAttempt < 2) {
		result = FutureUtils.completedExceptionally(new FlinkException("Could not resolve the leader gateway."));
	} else {
		result = CompletableFuture.completedFuture(rpcGateway);
	}

	retrievalAttempt++;

	return result;
}
 

@Override
public CompletableFuture<Acknowledge> submitJob(JobGraph jobGraph, Time timeout) {
	final CompletableFuture<Acknowledge> acknowledgeCompletableFuture = super.submitJob(jobGraph, timeout);

	acknowledgeCompletableFuture.whenComplete(
		(Acknowledge ignored, Throwable throwable) -> {
			if (throwable != null) {
				onFatalError(new FlinkException(
					"Failed to submit job " + jobGraph.getJobID() + " in job mode.",
					throwable));
			}
		});

	return acknowledgeCompletableFuture;
}
 

@Test
public void testPayloadAssignmentAfterRelease() {
	final SingleLogicalSlot singleLogicalSlot = createSingleLogicalSlot();
	final DummyPayload dummyPayload = new DummyPayload();

	singleLogicalSlot.releaseSlot(new FlinkException("Test exception"));

	assertThat(singleLogicalSlot.tryAssignPayload(dummyPayload), is(false));
}
 

/**
 * Tests that failing an allocation fails the pending slot request.
 */
@Test
public void testFailingAllocationFailsPendingSlotRequests() throws Exception {

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {
		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
		resourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final CompletableFuture<LogicalSlot> slotFuture = allocateSlot(scheduler, new SlotRequestId());

		final AllocationID allocationId = allocationIdFuture.get();

		assertThat(slotFuture.isDone(), is(false));

		final FlinkException cause = new FlinkException("Fail pending slot request failure.");
		final Optional<ResourceID> responseFuture = slotPool.failAllocation(allocationId, cause);

		assertThat(responseFuture.isPresent(), is(false));

		try {
			slotFuture.get();
			fail("Expected a slot allocation failure.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.stripExecutionException(ee), equalTo(cause));
		}
	}
}