Java Code Examples for org.apache.flink.runtime.dispatcher.DispatcherGateway

The following examples show how to use org.apache.flink.runtime.dispatcher.DispatcherGateway. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: MiniCluster.java    License: Apache License 2.0 6 votes vote down vote up
public CompletableFuture<JobSubmissionResult> submitJob(JobGraph jobGraph) {
	final CompletableFuture<DispatcherGateway> dispatcherGatewayFuture = getDispatcherGatewayFuture();

	// we have to allow queued scheduling in Flip-6 mode because we need to request slots
	// from the ResourceManager
	jobGraph.setAllowQueuedScheduling(true);

	final CompletableFuture<InetSocketAddress> blobServerAddressFuture = createBlobServerAddress(dispatcherGatewayFuture);

	final CompletableFuture<Void> jarUploadFuture = uploadAndSetJobFiles(blobServerAddressFuture, jobGraph);

	final CompletableFuture<Acknowledge> acknowledgeCompletableFuture = jarUploadFuture
		.thenCombine(
			dispatcherGatewayFuture,
			(Void ack, DispatcherGateway dispatcherGateway) -> dispatcherGateway.submitJob(jobGraph, rpcTimeout))
		.thenCompose(Function.identity());

	return acknowledgeCompletableFuture.thenApply(
		(Acknowledge ignored) -> new JobSubmissionResult(jobGraph.getJobID()));
}
 
Example 2
Source Project: flink   Source File: ApplicationDispatcherBootstrap.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
CompletableFuture<Void> fixJobIdAndRunApplicationAsync(
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor) {

	final Optional<String> configuredJobId =
			configuration.getOptional(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);

	if (!HighAvailabilityMode.isHighAvailabilityModeActivated(configuration) && !configuredJobId.isPresent()) {
		return runApplicationAsync(dispatcher, scheduledExecutor, false);
	}

	if (!configuredJobId.isPresent()) {
		configuration.set(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, ZERO_JOB_ID.toHexString());
	}
	return runApplicationAsync(dispatcher, scheduledExecutor, true);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: JobSubmitHandler.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<JobGraph> uploadJobGraphFiles(
		DispatcherGateway gateway,
		CompletableFuture<JobGraph> jobGraphFuture,
		Collection<Path> jarFiles,
		Collection<Tuple2<String, Path>> artifacts,
		Configuration configuration) {
	CompletableFuture<Integer> blobServerPortFuture = gateway.getBlobServerPort(timeout);

	return jobGraphFuture.thenCombine(blobServerPortFuture, (JobGraph jobGraph, Integer blobServerPort) -> {
		final InetSocketAddress address = new InetSocketAddress(gateway.getHostname(), blobServerPort);
		try {
			ClientUtils.uploadJobGraphFiles(jobGraph, jarFiles, artifacts, () -> new BlobClient(address, configuration));
		} catch (FlinkException e) {
			throw new CompletionException(new RestHandlerException(
				"Could not upload job files.",
				HttpResponseStatus.INTERNAL_SERVER_ERROR,
				e));
		}
		return jobGraph;
	});
}
 
Example 4
Source Project: Flink-CEPplus   Source File: AbstractMetricsHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(
		new QueryScopeInfo.JobManagerQueryScopeInfo(),
		TEST_METRIC_NAME,
		TEST_METRIC_VALUE));

	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);

	testMetricsHandler = new TestMetricsHandler(
		new GatewayRetriever<DispatcherGateway>() {
			@Override
			public CompletableFuture<DispatcherGateway> getFuture() {
				return CompletableFuture.completedFuture(mockDispatcherGateway);
			}
		},
		Time.milliseconds(50),
		Collections.emptyMap(),
		new TestMetricsHeaders(),
		mockMetricFetcher);
}
 
Example 5
Source Project: Flink-CEPplus   Source File: MetricsHandlerTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	this.leaderRetriever = new GatewayRetriever<DispatcherGateway>() {
		@Override
		public CompletableFuture<DispatcherGateway> getFuture() {
			return CompletableFuture.completedFuture(mockDispatcherGateway);
		}
	};
	this.pathParameters = getPathParameters();
	this.metricsHandler = getMetricsHandler();

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(getQueryScopeInfo(), TEST_METRIC_NAME,
		TEST_METRIC_VALUE));
	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);
}
 
Example 6
Source Project: Flink-CEPplus   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 
Example 7
Source Project: flink   Source File: AbstractMetricsHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() {
	MockitoAnnotations.initMocks(this);

	final MetricStore metricStore = new MetricStore();
	metricStore.add(new MetricDump.CounterDump(
		new QueryScopeInfo.JobManagerQueryScopeInfo(),
		TEST_METRIC_NAME,
		TEST_METRIC_VALUE));

	when(mockMetricFetcher.getMetricStore()).thenReturn(metricStore);

	testMetricsHandler = new TestMetricsHandler(
		new GatewayRetriever<DispatcherGateway>() {
			@Override
			public CompletableFuture<DispatcherGateway> getFuture() {
				return CompletableFuture.completedFuture(mockDispatcherGateway);
			}
		},
		Time.milliseconds(50),
		Collections.emptyMap(),
		new TestMetricsHeaders(),
		mockMetricFetcher);
}
 
Example 8
Source Project: flink   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 
Example 9
Source Project: Flink-CEPplus   Source File: RestClusterClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected CompletableFuture<JobExecutionResultResponseBody> handleRequest(
		@Nonnull HandlerRequest<EmptyRequestBody, JobMessageParameters> request,
		@Nonnull DispatcherGateway gateway) throws RestHandlerException {
	if (jobExecutionResults.hasNext()) {
		lastJobExecutionResult = jobExecutionResults.next();
	}
	checkState(lastJobExecutionResult != null);
	if (lastJobExecutionResult instanceof JobExecutionResultResponseBody) {
		return CompletableFuture.completedFuture((JobExecutionResultResponseBody) lastJobExecutionResult);
	} else if (lastJobExecutionResult instanceof RestHandlerException) {
		return FutureUtils.completedExceptionally((RestHandlerException) lastJobExecutionResult);
	} else {
		throw new AssertionError();
	}
}
 
Example 10
Source Project: flink   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSerializationFailureHandling() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance()), mockGateway);
		Assert.fail();
	} catch (RestHandlerException rhe) {
		Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rhe.getHttpResponseStatus());
	}
}
 
Example 11
Source Project: flink   Source File: EmbeddedExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private static CompletableFuture<JobID> submitJob(
		final Configuration configuration,
		final DispatcherGateway dispatcherGateway,
		final JobGraph jobGraph,
		final Time rpcTimeout) {
	checkNotNull(jobGraph);

	LOG.info("Submitting Job with JobId={}.", jobGraph.getJobID());

	return dispatcherGateway
			.getBlobServerPort(rpcTimeout)
			.thenApply(blobServerPort -> new InetSocketAddress(dispatcherGateway.getHostname(), blobServerPort))
			.thenCompose(blobServerAddress -> {

				try {
					ClientUtils.extractAndUploadJobGraphFiles(jobGraph, () -> new BlobClient(blobServerAddress, configuration));
				} catch (FlinkException e) {
					throw new CompletionException(e);
				}

				return dispatcherGateway.submitJob(jobGraph, rpcTimeout);
			}).thenApply(ack -> jobGraph.getJobID());
}
 
Example 12
Source Project: flink   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSerializationFailureHandling() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	DispatcherGateway mockGateway = new TestingDispatcherGateway.Builder()
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.toString(), Collections.emptyList(), Collections.emptyList());

	try {
		handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance()), mockGateway);
		Assert.fail();
	} catch (RestHandlerException rhe) {
		Assert.assertEquals(HttpResponseStatus.BAD_REQUEST, rhe.getHttpResponseStatus());
	}
}
 
Example 13
Source Project: flink   Source File: JobSubmitHandlerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSuccessfulJobSubmission() throws Exception {
	final Path jobGraphFile = TEMPORARY_FOLDER.newFile().toPath();
	try (ObjectOutputStream objectOut = new ObjectOutputStream(Files.newOutputStream(jobGraphFile))) {
		objectOut.writeObject(new JobGraph("testjob"));
	}

	TestingDispatcherGateway.Builder builder = new TestingDispatcherGateway.Builder();
	builder
		.setBlobServerPort(blobServer.getPort())
		.setSubmitFunction(jobGraph -> CompletableFuture.completedFuture(Acknowledge.get()))
		.setHostname("localhost");
	DispatcherGateway mockGateway = builder.build();

	JobSubmitHandler handler = new JobSubmitHandler(
		() -> CompletableFuture.completedFuture(mockGateway),
		RpcUtils.INF_TIMEOUT,
		Collections.emptyMap(),
		TestingUtils.defaultExecutor(),
		configuration);

	JobSubmitRequestBody request = new JobSubmitRequestBody(jobGraphFile.getFileName().toString(), Collections.emptyList(), Collections.emptyList());

	handler.handleRequest(new HandlerRequest<>(request, EmptyMessageParameters.getInstance(), Collections.emptyMap(), Collections.emptyMap(), Collections.singleton(jobGraphFile.toFile())), mockGateway)
		.get();
}
 
Example 14
Source Project: flink   Source File: JobSubmitHandler.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<JobGraph> uploadJobGraphFiles(
		DispatcherGateway gateway,
		CompletableFuture<JobGraph> jobGraphFuture,
		Collection<Path> jarFiles,
		Collection<Tuple2<String, Path>> artifacts,
		Configuration configuration) {
	CompletableFuture<Integer> blobServerPortFuture = gateway.getBlobServerPort(timeout);

	return jobGraphFuture.thenCombine(blobServerPortFuture, (JobGraph jobGraph, Integer blobServerPort) -> {
		final InetSocketAddress address = new InetSocketAddress(gateway.getHostname(), blobServerPort);
		try {
			ClientUtils.uploadJobGraphFiles(jobGraph, jarFiles, artifacts, () -> new BlobClient(address, configuration));
		} catch (FlinkException e) {
			throw new CompletionException(new RestHandlerException(
				"Could not upload job files.",
				HttpResponseStatus.INTERNAL_SERVER_ERROR,
				e));
		}
		return jobGraph;
	});
}
 
Example 15
Source Project: flink   Source File: RestClusterClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected CompletableFuture<JobExecutionResultResponseBody> handleRequest(
		@Nonnull HandlerRequest<EmptyRequestBody, JobMessageParameters> request,
		@Nonnull DispatcherGateway gateway) throws RestHandlerException {
	if (jobExecutionResults.hasNext()) {
		lastJobExecutionResult = jobExecutionResults.next();
	}
	checkState(lastJobExecutionResult != null);
	if (lastJobExecutionResult instanceof JobExecutionResultResponseBody) {
		return CompletableFuture.completedFuture((JobExecutionResultResponseBody) lastJobExecutionResult);
	} else if (lastJobExecutionResult instanceof RestHandlerException) {
		return FutureUtils.completedExceptionally((RestHandlerException) lastJobExecutionResult);
	} else {
		throw new AssertionError();
	}
}
 
Example 16
Source Project: flink   Source File: JarRunHandler.java    License: Apache License 2.0 6 votes vote down vote up
public JarRunHandler(
		final GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<JarRunRequestBody, JarRunResponseBody, JarRunMessageParameters> messageHeaders,
		final Path jarDir,
		final Configuration configuration,
		final Executor executor,
		final Supplier<ApplicationRunner> applicationRunnerSupplier) {
	super(leaderRetriever, timeout, responseHeaders, messageHeaders);

	this.jarDir = requireNonNull(jarDir);
	this.configuration = requireNonNull(configuration);
	this.executor = requireNonNull(executor);

	this.applicationRunner = applicationRunnerSupplier.get();
}
 
Example 17
Source Project: flink   Source File: JarRunHandler.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected CompletableFuture<JarRunResponseBody> handleRequest(
		@Nonnull final HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request,
		@Nonnull final DispatcherGateway gateway) throws RestHandlerException {

	final Configuration effectiveConfiguration = new Configuration(configuration);
	effectiveConfiguration.set(DeploymentOptions.ATTACHED, false);
	effectiveConfiguration.set(DeploymentOptions.TARGET, EmbeddedExecutor.NAME);

	final JarHandlerContext context = JarHandlerContext.fromRequest(request, jarDir, log);
	context.applyToConfiguration(effectiveConfiguration);
	SavepointRestoreSettings.toConfiguration(getSavepointRestoreSettings(request), effectiveConfiguration);

	final PackagedProgram program = context.toPackagedProgram(effectiveConfiguration);

	return CompletableFuture
			.supplyAsync(() -> applicationRunner.run(gateway, program, effectiveConfiguration), executor)
			.thenApply(jobIds -> {
				if (jobIds.isEmpty()) {
					throw new CompletionException(new ProgramInvocationException("No jobs submitted."));
				}
				return new JarRunResponseBody(jobIds.get(0));
			});
}
 
Example 18
Source Project: flink   Source File: RestClusterClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
protected CompletableFuture<ClientCoordinationResponseBody> handleRequest(@Nonnull HandlerRequest<ClientCoordinationRequestBody, ClientCoordinationMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
	try {
		TestCoordinationRequest req =
			(TestCoordinationRequest) request
				.getRequestBody()
				.getSerializedCoordinationRequest()
				.deserializeValue(getClass().getClassLoader());
		TestCoordinationResponse resp = new TestCoordinationResponse(req.payload);
		return CompletableFuture.completedFuture(
			new ClientCoordinationResponseBody(
				new SerializedValue<>(resp)));
	} catch (Exception e) {
		return FutureUtils.completedExceptionally(e);
	}
}
 
Example 19
Source Project: flink   Source File: ApplicationDispatcherBootstrap.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Runs the user program entrypoint and completes the given {@code jobIdsFuture} with the {@link
 * JobID JobIDs} of the submitted jobs.
 *
 * <p>This should be executed in a separate thread (or task).
 */
private void runApplicationEntryPoint(
		final CompletableFuture<List<JobID>> jobIdsFuture,
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor,
		final boolean enforceSingleJobExecution) {
	try {
		final List<JobID> applicationJobIds =
				new ArrayList<>(getRecoveredJobIds(recoveredJobs));

		final PipelineExecutorServiceLoader executorServiceLoader =
				new EmbeddedExecutorServiceLoader(
						applicationJobIds, dispatcher, scheduledExecutor);

		ClientUtils.executeProgram(
				executorServiceLoader,
				configuration,
				application,
				enforceSingleJobExecution,
				true /* suppress sysout */);

		if (applicationJobIds.isEmpty()) {
			jobIdsFuture.completeExceptionally(
					new ApplicationExecutionException(
							"The application contains no execute() calls."));
		} else {
			jobIdsFuture.complete(applicationJobIds);
		}
	} catch (Throwable t) {
		jobIdsFuture.completeExceptionally(
				new ApplicationExecutionException("Could not execute application.", t));
	}
}
 
Example 20
private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException {
	FutureUtils.retrySuccessfulWithDelay(
		() -> dispatcherGateway.requestClusterOverview(timeout),
		Time.milliseconds(50L),
		Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())),
		clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= 1 &&
			clusterOverview.getNumSlotsAvailable() == 0 &&
			clusterOverview.getNumSlotsTotal() == 2,
		TestingUtils.defaultScheduledExecutor())
		.get();
}
 
Example 21
Source Project: flink   Source File: MiniCluster.java    License: Apache License 2.0 5 votes vote down vote up
private CompletableFuture<InetSocketAddress> createBlobServerAddress(final CompletableFuture<DispatcherGateway> dispatcherGatewayFuture) {
	return dispatcherGatewayFuture.thenApply(dispatcherGateway ->
			dispatcherGateway
				.getBlobServerPort(rpcTimeout)
				.thenApply(blobServerPort -> new InetSocketAddress(dispatcherGateway.getHostname(), blobServerPort)))
		.thenCompose(Function.identity());
}
 
Example 22
Source Project: Flink-CEPplus   Source File: JarRunHandler.java    License: Apache License 2.0 5 votes vote down vote up
public JarRunHandler(
		final GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
		final Time timeout,
		final Map<String, String> responseHeaders,
		final MessageHeaders<JarRunRequestBody, JarRunResponseBody, JarRunMessageParameters> messageHeaders,
		final Path jarDir,
		final Configuration configuration,
		final Executor executor) {
	super(leaderRetriever, timeout, responseHeaders, messageHeaders);

	this.jarDir = requireNonNull(jarDir);
	this.configuration = requireNonNull(configuration);
	this.executor = requireNonNull(executor);
}
 
Example 23
Source Project: flink   Source File: DetachedApplicationRunner.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<JobID> run(final DispatcherGateway dispatcherGateway, final PackagedProgram program, final Configuration configuration) {
	checkNotNull(dispatcherGateway);
	checkNotNull(program);
	checkNotNull(configuration);
	return tryExecuteJobs(dispatcherGateway, program, configuration);
}
 
Example 24
Source Project: flink   Source File: WebMonitorUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Loads the {@link WebMonitorExtension} which enables web submission.
 *
 * @param leaderRetriever to retrieve the leader
 * @param timeout for asynchronous requests
 * @param responseHeaders for the web submission handlers
 * @param localAddressFuture of the underlying REST server endpoint
 * @param uploadDir where the web submission handler store uploaded jars
 * @param executor to run asynchronous operations
 * @param configuration used to instantiate the web submission extension
 * @return Web submission extension
 * @throws FlinkException if the web submission extension could not be loaded
 */
public static WebMonitorExtension loadWebSubmissionExtension(
		GatewayRetriever<? extends DispatcherGateway> leaderRetriever,
		Time timeout,
		Map<String, String> responseHeaders,
		CompletableFuture<String> localAddressFuture,
		java.nio.file.Path uploadDir,
		Executor executor,
		Configuration configuration) throws FlinkException {

	if (isFlinkRuntimeWebInClassPath()) {
		try {
			final Constructor<?> webSubmissionExtensionConstructor = Class
				.forName("org.apache.flink.runtime.webmonitor.WebSubmissionExtension")
				.getConstructor(
					Configuration.class,
					GatewayRetriever.class,
					Map.class,
					CompletableFuture.class,
					java.nio.file.Path.class,
					Executor.class,
					Time.class);

			return (WebMonitorExtension) webSubmissionExtensionConstructor.newInstance(
				configuration,
				leaderRetriever,
				responseHeaders,
				localAddressFuture,
				uploadDir,
				executor,
				timeout);
		} catch (ClassNotFoundException | NoSuchMethodException | InstantiationException | InvocationTargetException | IllegalAccessException e) {
			throw new FlinkException("Could not load web submission extension.", e);
		}
	} else {
		throw new FlinkException("The module flink-runtime-web could not be found in the class path. Please add " +
			"this jar in order to enable web based job submission.");
	}
}
 
Example 25
Source Project: flink   Source File: EmbeddedJobClient.java    License: Apache License 2.0 5 votes vote down vote up
public EmbeddedJobClient(
		final JobID jobId,
		final DispatcherGateway dispatcherGateway,
		final ScheduledExecutor retryExecutor,
		final Time rpcTimeout) {
	this.jobId = checkNotNull(jobId);
	this.dispatcherGateway = checkNotNull(dispatcherGateway);
	this.retryExecutor = checkNotNull(retryExecutor);
	this.timeout = checkNotNull(rpcTimeout);
}
 
Example 26
Source Project: Flink-CEPplus   Source File: RestEndpointFactory.java    License: Apache License 2.0 5 votes vote down vote up
WebMonitorEndpoint<T> createRestEndpoint(
Configuration configuration,
LeaderGatewayRetriever<DispatcherGateway> dispatcherGatewayRetriever,
LeaderGatewayRetriever<ResourceManagerGateway> resourceManagerGatewayRetriever,
TransientBlobService transientBlobService,
ExecutorService executor,
MetricFetcher metricFetcher,
LeaderElectionService leaderElectionService,
FatalErrorHandler fatalErrorHandler) throws Exception;
 
Example 27
Source Project: Flink-CEPplus   Source File: JobSubmitHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected CompletableFuture<JobSubmitResponseBody> handleRequest(@Nonnull HandlerRequest<JobSubmitRequestBody, EmptyMessageParameters> request, @Nonnull DispatcherGateway gateway) throws RestHandlerException {
	final Collection<File> uploadedFiles = request.getUploadedFiles();
	final Map<String, Path> nameToFile = uploadedFiles.stream().collect(Collectors.toMap(
		File::getName,
		Path::fromLocalFile
	));

	if (uploadedFiles.size() != nameToFile.size()) {
		throw new RestHandlerException(
			String.format("The number of uploaded files was %s than the expected count. Expected: %s Actual %s",
				uploadedFiles.size() < nameToFile.size() ? "lower" : "higher",
				nameToFile.size(),
				uploadedFiles.size()),
			HttpResponseStatus.BAD_REQUEST
		);
	}

	final JobSubmitRequestBody requestBody = request.getRequestBody();

	if (requestBody.jobGraphFileName == null) {
		throw new RestHandlerException(
			String.format("The %s field must not be omitted or be null.",
				JobSubmitRequestBody.FIELD_NAME_JOB_GRAPH),
			HttpResponseStatus.BAD_REQUEST);
	}

	CompletableFuture<JobGraph> jobGraphFuture = loadJobGraph(requestBody, nameToFile);

	Collection<Path> jarFiles = getJarFilesToUpload(requestBody.jarFileNames, nameToFile);

	Collection<Tuple2<String, Path>> artifacts = getArtifactFilesToUpload(requestBody.artifactFileNames, nameToFile);

	CompletableFuture<JobGraph> finalizedJobGraphFuture = uploadJobGraphFiles(gateway, jobGraphFuture, jarFiles, artifacts, configuration);

	CompletableFuture<Acknowledge> jobSubmissionFuture = finalizedJobGraphFuture.thenCompose(jobGraph -> gateway.submitJob(jobGraph, timeout));

	return jobSubmissionFuture.thenCombine(jobGraphFuture,
		(ack, jobGraph) -> new JobSubmitResponseBody("/jobs/" + jobGraph.getJobID()));
}
 
Example 28
Source Project: Flink-CEPplus   Source File: AbstractMetricsHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
private TestMetricsHandler(
	GatewayRetriever<DispatcherGateway> leaderRetriever,
	Time timeout,
	Map<String, String> headers,
	MessageHeaders<EmptyRequestBody,
		MetricCollectionResponseBody,
		TestMessageParameters> messageHeaders,
	MetricFetcher metricFetcher) {

	super(leaderRetriever, timeout, headers, messageHeaders, metricFetcher);
}
 
Example 29
Source Project: flink   Source File: TestingDispatcherLeaderProcess.java    License: Apache License 2.0 5 votes vote down vote up
private TestingDispatcherLeaderProcess(
		UUID leaderSessionId,
		CompletableFuture<DispatcherGateway> dispatcherGatewayFuture,
		Consumer<Void> startConsumer,
		Supplier<CompletableFuture<Void>> closeAsyncSupplier,
		CompletableFuture<String> confirmLeaderSessionFuture,
		CompletableFuture<ApplicationStatus> shutDownFuture) {
	this.leaderSessionId = leaderSessionId;
	this.dispatcherGatewayFuture = dispatcherGatewayFuture;
	this.startConsumer = startConsumer;
	this.closeAsyncSupplier = closeAsyncSupplier;
	this.confirmLeaderSessionFuture = confirmLeaderSessionFuture;
	this.shutDownFuture = shutDownFuture;
}
 
Example 30
Source Project: flink   Source File: EmbeddedExecutorServiceLoader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates an {@link EmbeddedExecutorServiceLoader}.
 * @param submittedJobIds a list that is going to be filled by the {@link EmbeddedExecutor} with the job ids of the
 *                        new jobs that will be submitted. This is essentially used to return the submitted job ids
 *                        to the caller.
 * @param dispatcherGateway the dispatcher of the cluster which is going to be used to submit jobs.
 */
public EmbeddedExecutorServiceLoader(
		final Collection<JobID> submittedJobIds,
		final DispatcherGateway dispatcherGateway,
		final ScheduledExecutor retryExecutor) {
	this.submittedJobIds = checkNotNull(submittedJobIds);
	this.dispatcherGateway = checkNotNull(dispatcherGateway);
	this.retryExecutor = checkNotNull(retryExecutor);
}