Java Code Examples for org.apache.flink.client.ClientUtils

The following examples show how to use org.apache.flink.client.ClientUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: AbstractCustomCommandLine.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 
Example 2
Source Project: flink   Source File: AbstractCustomCommandLine.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 
Example 3
Source Project: flink   Source File: AbstractOperatorRestoreTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private void restoreJob(ClusterClient<?> clusterClient, Deadline deadline, String savepointPath) throws Exception {
	JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE);
	jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState));

	assertNotNull("Job doesn't have a JobID.", jobToRestore.getJobID());

	ClientUtils.submitJob(clusterClient, jobToRestore);

	CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay(
		() -> clusterClient.getJobStatus(jobToRestore.getJobID()),
		Time.milliseconds(50),
		deadline,
		(jobStatus) -> jobStatus == JobStatus.FINISHED,
		TestingUtils.defaultScheduledExecutor());
	assertEquals(
		JobStatus.FINISHED,
		jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
}
 
Example 4
Source Project: flink   Source File: SavepointITCase.java    License: Apache License 2.0 6 votes vote down vote up
private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		ClientUtils.submitJob(client, jobGraph);

		StatefulCounter.getProgressLatch().await();

		return client.cancelWithSavepoint(jobId, null).get();
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}
 
Example 5
Source Project: flink   Source File: SavepointTestBase.java    License: Apache License 2.0 6 votes vote down vote up
public <T> String takeSavepoint(Collection<T> data, Function<SourceFunction<T>, StreamExecutionEnvironment> jobGraphFactory) throws Exception {

		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().disableClosureCleaner();

		WaitingSource<T> waitingSource = createSource(data);

		JobGraph jobGraph = jobGraphFactory.apply(waitingSource).getStreamGraph().getJobGraph();
		JobID jobId = jobGraph.getJobID();

		ClusterClient<?> client = miniClusterResource.getClusterClient();

		try {
			JobSubmissionResult result = ClientUtils.submitJob(client, jobGraph);

			return CompletableFuture
				.runAsync(waitingSource::awaitSource)
				.thenCompose(ignore -> triggerSavepoint(client, result.getJobID()))
				.get(5, TimeUnit.MINUTES);
		} catch (Exception e) {
			throw new RuntimeException("Failed to take savepoint", e);
		} finally {
			client.cancel(jobId);
		}
	}
 
Example 6
Source Project: flink   Source File: DetachedApplicationRunner.java    License: Apache License 2.0 6 votes vote down vote up
private List<JobID> tryExecuteJobs(final DispatcherGateway dispatcherGateway, final PackagedProgram program, final Configuration configuration) {
	configuration.set(DeploymentOptions.ATTACHED, false);

	final List<JobID> applicationJobIds = new ArrayList<>();
	final PipelineExecutorServiceLoader executorServiceLoader =
			new WebSubmissionExecutorServiceLoader(applicationJobIds, dispatcherGateway);

	try {
		ClientUtils.executeProgram(executorServiceLoader, configuration, program, enforceSingleJobExecution, true);
	} catch (ProgramInvocationException e) {
		LOG.warn("Could not execute application: ", e);
		throw new FlinkRuntimeException("Could not execute application.", e);
	}

	return applicationJobIds;
}
 
Example 7
Source Project: flink   Source File: ClientTest.java    License: Apache License 2.0 6 votes vote down vote up
private void launchMultiExecuteJob(final boolean enforceSingleJobExecution) throws ProgramInvocationException {
	try (final ClusterClient<?> clusterClient =
				new MiniClusterClient(new Configuration(), MINI_CLUSTER_RESOURCE.getMiniCluster())) {

		final PackagedProgram program = PackagedProgram.newBuilder()
				.setEntryPointClassName(TestMultiExecute.class.getName())
				.build();

		final Configuration configuration = fromPackagedProgram(program, 1, false);

		ClientUtils.executeProgram(
				new TestExecutorServiceLoader(clusterClient, plan),
				configuration,
				program,
				enforceSingleJobExecution,
				false);
	}
}
 
Example 8
Source Project: flink   Source File: ClientTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test verifies that the local execution environment cannot be created when
 * the program is submitted through a client.
 */
@Test
public void tryLocalExecution() throws ProgramInvocationException, ProgramMissingJobException {
	PackagedProgram packagedProgramMock = mock(PackagedProgram.class);

	when(packagedProgramMock.getUserCodeClassLoader())
			.thenReturn(packagedProgramMock.getClass().getClassLoader());

	doAnswer(new Answer<Void>() {
		@Override
		public Void answer(InvocationOnMock invocation) throws Throwable {
			ExecutionEnvironment.createLocalEnvironment();
			return null;
		}
	}).when(packagedProgramMock).invokeInteractiveModeForExecution();

	try {
		final ClusterClient<?> client = new MiniClusterClient(new Configuration(), MINI_CLUSTER_RESOURCE.getMiniCluster());
		final Configuration configuration = fromPackagedProgram(packagedProgramMock, 1, true);
		ClientUtils.executeProgram(new TestExecutorServiceLoader(client, plan), configuration, packagedProgramMock, false, false);
		fail("Creating the local execution environment should not be possible");
	}
	catch (InvalidProgramException e) {
		// that is what we want
	}
}
 
Example 9
Source Project: flink   Source File: RestClusterClientTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that we can submit a jobGraph in detached mode.
 */
@Test
public void testDetachedJobSubmission() throws Exception {

	final TestJobSubmitHandler testJobSubmitHandler = new TestJobSubmitHandler();

	try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(
		testJobSubmitHandler)) {
		RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort());

		try {
			final JobSubmissionResult jobSubmissionResult = ClientUtils.submitJob(restClusterClient, jobGraph);

			// if the detached mode didn't work, then we would not reach this point because the execution result
			// retrieval would have failed.
			assertThat(jobSubmissionResult, is(instanceOf(DetachedJobExecutionResult.class)));
			assertThat(jobSubmissionResult.getJobID(), is(jobId));
		} finally {
			restClusterClient.close();
		}
	}

}
 
Example 10
Source Project: flink   Source File: JobRetrievalITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJobRetrieval() throws Exception {
	final JobID jobID = new JobID();

	final JobVertex imalock = new JobVertex("imalock");
	imalock.setInvokableClass(SemaphoreInvokable.class);

	final JobGraph jobGraph = new JobGraph(jobID, "testjob", imalock);

	// acquire the lock to make sure that the job cannot complete until the job client
	// has been attached in resumingThread
	lock.acquire();

	ClientUtils.submitJob(client, jobGraph);

	final CheckedThread resumingThread = new CheckedThread("Flink-Job-Retriever") {
		@Override
		public void go() throws Exception {
			assertNotNull(client.requestJobResult(jobID).get());
		}
	};

	// wait until the job is running
	while (client.listJobs().get().isEmpty()) {
		Thread.sleep(50);
	}

	// kick off resuming
	resumingThread.start();

	// wait for client to connect
	while (resumingThread.getState() != Thread.State.WAITING) {
		Thread.sleep(10);
	}

	// client has connected, we can release the lock
	lock.release();

	resumingThread.sync();
}
 
Example 11
Source Project: flink   Source File: CancelingTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected void runAndCancelJob(Plan plan, final int msecsTillCanceling, int maxTimeTillCanceled) throws Exception {
	// submit job
	final JobGraph jobGraph = getJobGraph(plan);

	final long rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration).toMilliseconds();

	ClusterClient<?> client = CLUSTER.getClusterClient();
	JobSubmissionResult jobSubmissionResult = ClientUtils.submitJob(client, jobGraph);

	Deadline submissionDeadLine = new FiniteDuration(2, TimeUnit.MINUTES).fromNow();

	JobStatus jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(rpcTimeout, TimeUnit.MILLISECONDS);
	while (jobStatus != JobStatus.RUNNING && submissionDeadLine.hasTimeLeft()) {
		Thread.sleep(50);
		jobStatus = client.getJobStatus(jobSubmissionResult.getJobID()).get(rpcTimeout, TimeUnit.MILLISECONDS);
	}
	if (jobStatus != JobStatus.RUNNING) {
		Assert.fail("Job not in state RUNNING.");
	}

	Thread.sleep(msecsTillCanceling);

	client.cancel(jobSubmissionResult.getJobID()).get();

	Deadline cancelDeadline = new FiniteDuration(maxTimeTillCanceled, TimeUnit.MILLISECONDS).fromNow();

	JobStatus jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(rpcTimeout, TimeUnit.MILLISECONDS);
	while (jobStatusAfterCancel != JobStatus.CANCELED && cancelDeadline.hasTimeLeft()) {
		Thread.sleep(50);
		jobStatusAfterCancel = client.getJobStatus(jobSubmissionResult.getJobID()).get(rpcTimeout, TimeUnit.MILLISECONDS);
	}
	if (jobStatusAfterCancel != JobStatus.CANCELED) {
		Assert.fail("Failed to cancel job with ID " + jobSubmissionResult.getJobID() + '.');
	}
}
 
Example 12
Source Project: flink   Source File: NotifyCheckpointAbortedITCase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verify operators would be notified as checkpoint aborted.
 *
 * <p>The job would run with at least two checkpoints. The 1st checkpoint would fail due to add checkpoint to store,
 * and the 2nd checkpoint would decline by async checkpoint phase of 'DeclineSink'.
 *
 * <p>The job graph looks like:
 * NormalSource --> keyBy --> NormalMap --> DeclineSink
 */
@Test(timeout = TEST_TIMEOUT)
public void testNotifyCheckpointAborted() throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(200, CheckpointingMode.EXACTLY_ONCE);
	env.getCheckpointConfig().enableUnalignedCheckpoints(unalignedCheckpointEnabled);
	env.getCheckpointConfig().setTolerableCheckpointFailureNumber(1);
	env.disableOperatorChaining();
	env.setParallelism(1);

	final StateBackend failingStateBackend = new DeclineSinkFailingStateBackend(checkpointPath);
	env.setStateBackend(failingStateBackend);

	env.addSource(new NormalSource()).name("NormalSource")
		.keyBy((KeySelector<Tuple2<Integer, Integer>, Integer>) value -> value.f0)
		.transform("NormalMap", TypeInformation.of(Integer.class), new NormalMap())
		.transform(DECLINE_SINK_NAME, TypeInformation.of(Object.class), new DeclineSink());

	final ClusterClient<?> clusterClient = cluster.getClusterClient();
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();
	JobID jobID = jobGraph.getJobID();

	ClientUtils.submitJob(clusterClient, jobGraph);

	TestingCompletedCheckpointStore.addCheckpointLatch.await();
	TestingCompletedCheckpointStore.abortCheckpointLatch.trigger();

	verifyAllOperatorsNotifyAborted();
	resetAllOperatorsNotifyAbortedLatches();
	verifyAllOperatorsNotifyAbortedTimes(1);

	DeclineSink.waitLatch.trigger();
	verifyAllOperatorsNotifyAborted();
	verifyAllOperatorsNotifyAbortedTimes(2);

	clusterClient.cancel(jobID).get();
}
 
Example 13
Source Project: flink   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		ClientUtils.submitJob(client, graph);

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}
 
Example 14
Source Project: flink   Source File: ResumeCheckpointManuallyITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static String runJobAndGetExternalizedCheckpoint(StateBackend backend, File checkpointDir, @Nullable String externalCheckpoint, ClusterClient<?> client) throws Exception {
	JobGraph initialJobGraph = getJobGraph(backend, externalCheckpoint);
	NotifyingInfiniteTupleSource.countDownLatch = new CountDownLatch(PARALLELISM);

	ClientUtils.submitJob(client, initialJobGraph);

	// wait until all sources have been started
	NotifyingInfiniteTupleSource.countDownLatch.await();

	waitUntilExternalizedCheckpointCreated(checkpointDir, initialJobGraph.getJobID());
	client.cancel(initialJobGraph.getJobID()).get();
	waitUntilCanceled(initialJobGraph.getJobID(), client);

	return getExternalizedCheckpointCheckpointPath(checkpointDir, initialJobGraph.getJobID());
}
 
Example 15
Source Project: flink   Source File: StreamFaultToleranceTestBase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
 * followed by the checks in {@link #postSubmit}.
 */
@Test
public void runCheckpointedProgram() throws Exception {
	try {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(PARALLELISM);
		env.enableCheckpointing(500);
					env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));

		testProgram(env);

		JobGraph jobGraph = env.getStreamGraph().getJobGraph();
		try {
			ClientUtils.submitJobAndWaitForResult(cluster.getClusterClient(), jobGraph, getClass().getClassLoader()).getJobExecutionResult();
		} catch (ProgramInvocationException root) {
			Throwable cause = root.getCause();

			// search for nested SuccessExceptions
			int depth = 0;
			while (!(cause instanceof SuccessException)) {
				if (cause == null || depth++ == 20) {
					root.printStackTrace();
					fail("Test failed: " + root.getMessage());
				}
				else {
					cause = cause.getCause();
				}
			}
		}

		postSubmit();
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 16
Source Project: flink   Source File: RegionFailoverITCase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a simple job (Source -> Map) with multi regions could restore with operator state.
 *
 * <p>The last subtask of Map function in the 1st stream graph would fail {@code NUM_OF_RESTARTS} times,
 * and it will verify whether the restored state is identical to last completed checkpoint's.
 */
@Test(timeout = 60000)
public void testMultiRegionFailover() {
	try {
		JobGraph jobGraph = createJobGraph();
		ClusterClient<?> client = cluster.getClusterClient();
		ClientUtils.submitJobAndWaitForResult(client, jobGraph, RegionFailoverITCase.class.getClassLoader());
		verifyAfterJobExecuted();
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 17
Source Project: flink   Source File: NetworkStackThroughputITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void testProgram(
		final MiniClusterWithClientResource cluster,
		final int dataVolumeGb,
		final boolean useForwarder,
		final boolean isSlowSender,
		final boolean isSlowReceiver,
		final int parallelism) throws Exception {
	ClusterClient<?> client = cluster.getClusterClient();

	JobExecutionResult jer = ClientUtils.submitJobAndWaitForResult(
		client,
		createJobGraph(
			dataVolumeGb,
			useForwarder,
			isSlowSender,
			isSlowReceiver,
			parallelism),
		getClass().getClassLoader());

	long dataVolumeMbit = dataVolumeGb * 8192;
	long runtimeSecs = jer.getNetRuntime(TimeUnit.SECONDS);

	int mbitPerSecond = (int) (((double) dataVolumeMbit) / runtimeSecs);

	LOG.info(String.format("Test finished with throughput of %d MBit/s (runtime [secs]: %d, " +
		"data volume [gb/mbits]: %d/%d)", mbitPerSecond, runtimeSecs, dataVolumeGb, dataVolumeMbit));
}
 
Example 18
Source Project: flink   Source File: SavepointReaderITTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private String takeSavepoint(JobGraph jobGraph) throws Exception {
	SavepointSource.initializeForTest();

	ClusterClient<?> client = miniClusterResource.getClusterClient();
	JobID jobId = jobGraph.getJobID();

	Deadline deadline = Deadline.fromNow(Duration.ofMinutes(5));

	String dirPath = getTempDirPath(new AbstractID().toHexString());

	try {
		JobSubmissionResult result = ClientUtils.submitJob(client, jobGraph);

		boolean finished = false;
		while (deadline.hasTimeLeft()) {
			if (SavepointSource.isFinished()) {
				finished = true;

				break;
			}

			try {
				Thread.sleep(2L);
			} catch (InterruptedException ignored) {
				Thread.currentThread().interrupt();
			}
		}

		if (!finished) {
			Assert.fail("Failed to initialize state within deadline");
		}

		CompletableFuture<String> path = client.triggerSavepoint(result.getJobID(), dirPath);
		return path.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
	} finally {
		client.cancel(jobId).get();
	}
}
 
Example 19
Source Project: flink   Source File: SavepointWriterITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void validateBootstrap(String savepointPath) throws ProgramInvocationException {
	StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
	sEnv.setStateBackend(backend);

	CollectSink.accountList.clear();

	sEnv.fromCollection(accounts)
		.keyBy(acc -> acc.id)
		.flatMap(new UpdateAndGetAccount())
		.uid(ACCOUNT_UID)
		.addSink(new CollectSink());

	sEnv
		.fromCollection(currencyRates)
		.connect(sEnv.fromCollection(currencyRates).broadcast(descriptor))
		.process(new CurrencyValidationFunction())
		.uid(CURRENCY_UID)
		.addSink(new DiscardingSink<>());

	JobGraph jobGraph = sEnv.getStreamGraph().getJobGraph();
	jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));

	ClusterClient<?> client = miniClusterResource.getClusterClient();
	ClientUtils.submitJobAndWaitForResult(client, jobGraph, SavepointWriterITCase.class.getClassLoader());

	Assert.assertEquals("Unexpected output", 3, CollectSink.accountList.size());
}
 
Example 20
Source Project: flink   Source File: SavepointWriterITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void validateModification(String savepointPath) throws ProgramInvocationException {
	StreamExecutionEnvironment sEnv = StreamExecutionEnvironment.getExecutionEnvironment();
	sEnv.setStateBackend(backend);

	CollectSink.accountList.clear();

	DataStream<Account> stream = sEnv.fromCollection(accounts)
		.keyBy(acc -> acc.id)
		.flatMap(new UpdateAndGetAccount())
		.uid(ACCOUNT_UID);

	stream.addSink(new CollectSink());

	stream
		.map(acc -> acc.id)
		.map(new StatefulOperator())
		.uid(MODIFY_UID)
		.addSink(new DiscardingSink<>());

	JobGraph jobGraph = sEnv.getStreamGraph().getJobGraph();
	jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));

	ClusterClient<?> client = miniClusterResource.getClusterClient();
	ClientUtils.submitJobAndWaitForResult(client, jobGraph, SavepointWriterITCase.class.getClassLoader());

	Assert.assertEquals("Unexpected output", 3, CollectSink.accountList.size());
}
 
Example 21
Source Project: flink   Source File: ApplicationDispatcherBootstrap.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Runs the user program entrypoint and completes the given {@code jobIdsFuture} with the {@link
 * JobID JobIDs} of the submitted jobs.
 *
 * <p>This should be executed in a separate thread (or task).
 */
private void runApplicationEntryPoint(
		final CompletableFuture<List<JobID>> jobIdsFuture,
		final DispatcherGateway dispatcher,
		final ScheduledExecutor scheduledExecutor,
		final boolean enforceSingleJobExecution) {
	try {
		final List<JobID> applicationJobIds =
				new ArrayList<>(getRecoveredJobIds(recoveredJobs));

		final PipelineExecutorServiceLoader executorServiceLoader =
				new EmbeddedExecutorServiceLoader(
						applicationJobIds, dispatcher, scheduledExecutor);

		ClientUtils.executeProgram(
				executorServiceLoader,
				configuration,
				application,
				enforceSingleJobExecution,
				true /* suppress sysout */);

		if (applicationJobIds.isEmpty()) {
			jobIdsFuture.completeExceptionally(
					new ApplicationExecutionException(
							"The application contains no execute() calls."));
		} else {
			jobIdsFuture.complete(applicationJobIds);
		}
	} catch (Throwable t) {
		jobIdsFuture.completeExceptionally(
				new ApplicationExecutionException("Could not execute application.", t));
	}
}
 
Example 22
Source Project: flink   Source File: ClientTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test verifies correct job submission messaging logic and plan translation calls.
 */
@Test
public void shouldSubmitToJobClient() throws Exception {
	final ClusterClient<?> clusterClient = new MiniClusterClient(new Configuration(), MINI_CLUSTER_RESOURCE.getMiniCluster());
	JobGraph jobGraph = FlinkPipelineTranslationUtil.getJobGraph(
			plan,
			new Configuration(),
			1);

	jobGraph.addJars(Collections.emptyList());
	jobGraph.setClasspaths(Collections.emptyList());

	JobSubmissionResult result = ClientUtils.submitJob(clusterClient, jobGraph);
	assertNotNull(result);
}
 
Example 23
Source Project: flink   Source File: RestClusterClientTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJobSubmitCancel() throws Exception {
	TestJobSubmitHandler submitHandler = new TestJobSubmitHandler();
	TestJobCancellationHandler terminationHandler = new TestJobCancellationHandler();
	TestJobExecutionResultHandler testJobExecutionResultHandler =
		new TestJobExecutionResultHandler(
			JobExecutionResultResponseBody.created(new JobResult.Builder()
				.applicationStatus(ApplicationStatus.SUCCEEDED)
				.jobId(jobId)
				.netRuntime(Long.MAX_VALUE)
				.build()));

	try (TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(
		submitHandler,
		terminationHandler,
		testJobExecutionResultHandler)) {

		try (RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort())) {
			Assert.assertFalse(submitHandler.jobSubmitted);
			ClientUtils.submitJobAndWaitForResult(restClusterClient, jobGraph, ClassLoader.getSystemClassLoader());
			Assert.assertTrue(submitHandler.jobSubmitted);

			Assert.assertFalse(terminationHandler.jobCanceled);
			restClusterClient.cancel(jobId).get();
			Assert.assertTrue(terminationHandler.jobCanceled);
		}
	}
}
 
Example 24
Source Project: flink   Source File: RestClusterClientTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testJobSubmissionFailureThrowsProgramInvocationException() throws Exception {
	try (final TestRestServerEndpoint restServerEndpoint = createRestServerEndpoint(new SubmissionFailingHandler())) {
		RestClusterClient<?> restClusterClient = createRestClusterClient(restServerEndpoint.getServerAddress().getPort());

		try {
			ClientUtils.submitJobAndWaitForResult(restClusterClient, jobGraph, ClassLoader.getSystemClassLoader());
		} catch (final ProgramInvocationException expected) {
			// expected
		} finally {
			restClusterClient.close();
		}
	}
}
 
Example 25
Source Project: flink   Source File: AbstractQueryableStateTestBase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests simple value state queryable state instance. Each source emits
 * (subtaskIndex, 0)..(subtaskIndex, numElements) tuples, which are then
 * queried. The tests succeeds after each subtask index is queried with
 * value numElements (the latest element updated the state).
 */
@Test
public void testValueState() throws Exception {
	final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
	final long numElements = 1024L;

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(stateBackend);
	env.setParallelism(maxParallelism);
	// Very important, because cluster is shared between tests and we
	// don't explicitly check that all slots are available before
	// submitting.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));

	DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));

	// Value state
	ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", source.getType());

	source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {
		private static final long serialVersionUID = 7662520075515707428L;

		@Override
		public Integer getKey(Tuple2<Integer, Long> value) {
			return value.f0;
		}
	}).asQueryableState("hakuna", valueState);

	try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {

		final JobID jobId = autoCancellableJob.getJobId();
		final JobGraph jobGraph = autoCancellableJob.getJobGraph();

		ClientUtils.submitJob(clusterClient, jobGraph);
		executeValueQuery(deadline, client, jobId, "hakuna", valueState, numElements);
	}
}
 
Example 26
Source Project: flink   Source File: AbstractQueryableStateTestBase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests simple value state queryable state instance. Each source emits
 * (subtaskIndex, 0)..(subtaskIndex, numElements) tuples, which are then
 * queried. The tests succeeds after each subtask index is queried with
 * value numElements (the latest element updated the state).
 *
 * <p>This is the same as the simple value state test, but uses the API shortcut.
 */
@Test
public void testValueStateShortcut() throws Exception {
	final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
	final long numElements = 1024L;

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(stateBackend);
	env.setParallelism(maxParallelism);
	// Very important, because cluster is shared between tests and we
	// don't explicitly check that all slots are available before
	// submitting.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));

	DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));

	// Value state shortcut
	final QueryableStateStream<Integer, Tuple2<Integer, Long>> queryableState =
			source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {
				private static final long serialVersionUID = 9168901838808830068L;

				@Override
				public Integer getKey(Tuple2<Integer, Long> value) {
					return value.f0;
				}
			}).asQueryableState("matata");

	@SuppressWarnings("unchecked")
	final ValueStateDescriptor<Tuple2<Integer, Long>> stateDesc =
			(ValueStateDescriptor<Tuple2<Integer, Long>>) queryableState.getStateDescriptor();

	try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {

		final JobID jobId = autoCancellableJob.getJobId();
		final JobGraph jobGraph = autoCancellableJob.getJobGraph();

		ClientUtils.submitJob(clusterClient, jobGraph);
		executeValueQuery(deadline, client, jobId, "matata", stateDesc, numElements);
	}
}
 
Example 27
Source Project: flink   Source File: KafkaConsumerTestBase.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that the source can be properly canceled when reading empty partitions.
 */
public void runCancelingOnEmptyInputTest() throws Exception {
	final String topic = "cancelingOnEmptyInputTopic";

	final int parallelism = 3;
	createTestTopic(topic, parallelism, 1);

	final AtomicReference<Throwable> error = new AtomicReference<>();

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(parallelism);
	env.enableCheckpointing(100);

	Properties props = new Properties();
	props.putAll(standardProps);
	props.putAll(secureProps);
	FlinkKafkaConsumerBase<String> source = kafkaServer.getConsumer(topic, new SimpleStringSchema(), props);

	env.addSource(source).addSink(new DiscardingSink<String>());

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
	final JobID jobId = jobGraph.getJobID();

	final Runnable jobRunner = () -> {
		try {
			ClientUtils.submitJobAndWaitForResult(client, jobGraph, KafkaConsumerTestBase.class.getClassLoader());
		} catch (Throwable t) {
			LOG.error("Job Runner failed with exception", t);
			error.set(t);
		}
	};

	Thread runnerThread = new Thread(jobRunner, "program runner thread");
	runnerThread.start();

	// wait a bit before canceling
	Thread.sleep(2000);

	Throwable failueCause = error.get();
	if (failueCause != null) {
		failueCause.printStackTrace();
		Assert.fail("Test failed prematurely with: " + failueCause.getMessage());
	}
	// cancel
	client.cancel(jobId).get();

	// wait for the program to be done and validate that we failed with the right exception
	runnerThread.join();

	assertEquals(JobStatus.CANCELED, client.getJobStatus(jobId).get());

	deleteTestTopic(topic);
}
 
Example 28
Source Project: flink   Source File: ExecutionContext.java    License: Apache License 2.0 4 votes vote down vote up
private ExecutionContext(
		Environment environment,
		SessionContext originalSessionContext,
		@Nullable SessionState sessionState,
		List<URL> dependencies,
		Configuration flinkConfig,
		ClusterClientServiceLoader clusterClientServiceLoader,
		Options commandLineOptions,
		List<CustomCommandLine> availableCommandLines) throws FlinkException {
	this.environment = environment;
	this.originalSessionContext = originalSessionContext;

	this.flinkConfig = flinkConfig;

	if (containsPythonFunction(environment)) {
		dependencies = addPythonDependency(dependencies);
	}

	// create class loader
	classLoader = ClientUtils.buildUserCodeClassLoader(
		dependencies,
		Collections.emptyList(),
		this.getClass().getClassLoader(),
		flinkConfig);

	// Initialize the TableEnvironment.
	initializeTableEnvironment(sessionState);

	LOG.debug("Deployment descriptor: {}", environment.getDeployment());
	final CommandLine commandLine = createCommandLine(
			environment.getDeployment(),
			commandLineOptions);

	flinkConfig.addAll(createExecutionConfig(
			commandLine,
			commandLineOptions,
			availableCommandLines,
			dependencies));

	final ClusterClientServiceLoader serviceLoader = checkNotNull(clusterClientServiceLoader);
	clusterClientFactory = serviceLoader.getClusterClientFactory(flinkConfig);
	checkState(clusterClientFactory != null);

	clusterId = clusterClientFactory.getClusterId(flinkConfig);
	clusterSpec = clusterClientFactory.getClusterSpecification(flinkConfig);
}
 
Example 29
Source Project: flink   Source File: AccumulatorLiveITCase.java    License: Apache License 2.0 4 votes vote down vote up
private static void submitJobAndVerifyResults(JobGraph jobGraph) throws Exception {
	Deadline deadline = Deadline.now().plus(Duration.ofSeconds(30));

	final ClusterClient<?> client = MINI_CLUSTER_RESOURCE.getClusterClient();

	final CheckedThread submissionThread = new CheckedThread() {
		@Override
		public void go() throws Exception {
			ClientUtils.submitJobAndWaitForResult(client, jobGraph, AccumulatorLiveITCase.class.getClassLoader());
		}
	};

	submissionThread.start();

	try {
		NotifyingMapper.notifyLatch.await();

		FutureUtils.retrySuccessfulWithDelay(
			() -> {
				try {
					return CompletableFuture.completedFuture(client.getAccumulators(jobGraph.getJobID()).get());
				} catch (Exception e) {
					return FutureUtils.completedExceptionally(e);
				}
			},
			Time.milliseconds(20),
			deadline,
			accumulators -> accumulators.size() == 1
				&& accumulators.containsKey(ACCUMULATOR_NAME)
				&& (int) accumulators.get(ACCUMULATOR_NAME) == NUM_ITERATIONS,
			TestingUtils.defaultScheduledExecutor()
		).get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);

		NotifyingMapper.shutdownLatch.trigger();
	} finally {
		NotifyingMapper.shutdownLatch.trigger();

		// wait for the job to have terminated
		submissionThread.sync();
	}
}
 
Example 30
Source Project: flink   Source File: AbstractOperatorRestoreTestBase.java    License: Apache License 2.0 4 votes vote down vote up
private String migrateJob(ClusterClient<?> clusterClient, Deadline deadline) throws Throwable {

		URL savepointResource = AbstractOperatorRestoreTestBase.class.getClassLoader().getResource("operatorstate/" + getMigrationSavepointName());
		if (savepointResource == null) {
			throw new IllegalArgumentException("Savepoint file does not exist.");
		}
		JobGraph jobToMigrate = createJobGraph(ExecutionMode.MIGRATE);
		jobToMigrate.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointResource.getFile()));

		assertNotNull(jobToMigrate.getJobID());

		ClientUtils.submitJob(clusterClient, jobToMigrate);

		CompletableFuture<JobStatus> jobRunningFuture = FutureUtils.retrySuccessfulWithDelay(
			() -> clusterClient.getJobStatus(jobToMigrate.getJobID()),
			Time.milliseconds(50),
			deadline,
			(jobStatus) -> jobStatus == JobStatus.RUNNING,
			TestingUtils.defaultScheduledExecutor());
		assertEquals(
			JobStatus.RUNNING,
			jobRunningFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));

		// Trigger savepoint
		File targetDirectory = tmpFolder.newFolder();
		String savepointPath = null;

		// FLINK-6918: Retry cancel with savepoint message in case that StreamTasks were not running
		// TODO: The retry logic should be removed once the StreamTask lifecycle has been fixed (see FLINK-4714)
		while (deadline.hasTimeLeft() && savepointPath == null) {
			try {
				savepointPath = clusterClient.cancelWithSavepoint(
					jobToMigrate.getJobID(),
					targetDirectory.getAbsolutePath()).get();
			} catch (Exception e) {
				String exceptionString = ExceptionUtils.stringifyException(e);
				if (!PATTERN_CANCEL_WITH_SAVEPOINT_TOLERATED_EXCEPTIONS.matcher(exceptionString).find()) {
					throw e;
				}
			}
		}

		assertNotNull("Could not take savepoint.", savepointPath);

		CompletableFuture<JobStatus> jobCanceledFuture = FutureUtils.retrySuccessfulWithDelay(
			() -> clusterClient.getJobStatus(jobToMigrate.getJobID()),
			Time.milliseconds(50),
			deadline,
			(jobStatus) -> jobStatus == JobStatus.CANCELED,
			TestingUtils.defaultScheduledExecutor());
		assertEquals(
			JobStatus.CANCELED,
			jobCanceledFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));

		return savepointPath;
	}