org.apache.flink.streaming.api.graph.StreamGraph Java Examples

The following examples show how to use org.apache.flink.streaming.api.graph.StreamGraph. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IterateITCase.java    From Flink-CEPplus with Apache License 2.0 7 votes vote down vote up
@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
 
Example #2
Source File: StreamingJobGraphGeneratorNodeHashTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testUserProvidedHashing() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	List<String> userHashes = Arrays.asList("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");

	env.addSource(new NoOpSourceFunction(), "src").setUidHash(userHashes.get(0))
			.map(new NoOpMapFunction())
			.filter(new NoOpFilterFunction())
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce").setUidHash(userHashes.get(1));

	StreamGraph streamGraph = env.getStreamGraph();
	int idx = 1;
	for (JobVertex jobVertex : streamGraph.getJobGraph().getVertices()) {
		List<JobVertexID> idAlternatives = jobVertex.getIdAlternatives();
		Assert.assertEquals(idAlternatives.get(idAlternatives.size() - 1).toString(), userHashes.get(idx));
		--idx;
	}
}
 
Example #3
Source File: StreamContextEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	Preconditions.checkNotNull(jobName, "Streaming Job name should not be null.");

	StreamGraph streamGraph = this.getStreamGraph();
	streamGraph.setJobName(jobName);

	transformations.clear();

	// execute the programs
	if (ctx instanceof DetachedEnvironment) {
		LOG.warn("Job was executed in detached mode, the results will be available on completion.");
		((DetachedEnvironment) ctx).setDetachedPlan(streamGraph);
		return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE;
	} else {
		return ctx
			.getClient()
			.run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings())
			.getJobExecutionResult();
	}
}
 
Example #4
Source File: RemoteStreamEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Executes the job remotely.
 *
 * <p>This method can be used independent of the {@link StreamExecutionEnvironment} type.
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@PublicEvolving
public static JobExecutionResult executeRemotely(StreamExecutionEnvironment streamExecutionEnvironment,
	List<URL> jarFiles,
	String host,
	int port,
	Configuration clientConfiguration,
	List<URL> globalClasspaths,
	String jobName,
	SavepointRestoreSettings savepointRestoreSettings
) throws ProgramInvocationException {
	StreamGraph streamGraph = streamExecutionEnvironment.getStreamGraph();
	streamGraph.setJobName(jobName);
	return executeRemotely(streamGraph,
		streamExecutionEnvironment.getClass().getClassLoader(),
		streamExecutionEnvironment.getConfig(),
		jarFiles,
		host,
		port,
		clientConfiguration,
		globalClasspaths,
		savepointRestoreSettings);
}
 
Example #5
Source File: StreamPlanEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public JobExecutionResult execute(String jobName) throws Exception {

	StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);

	transformations.clear();

	if (env instanceof OptimizerPlanEnvironment) {
		((OptimizerPlanEnvironment) env).setPlan(streamGraph);
	} else if (env instanceof PreviewPlanEnvironment) {
		((PreviewPlanEnvironment) env).setPreview(streamGraph.getStreamingPlanAsJSON());
	}

	throw new OptimizerPlanEnvironment.ProgramAbortException();
}
 
Example #6
Source File: IterateITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
 
Example #7
Source File: RestartStrategyTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 
Example #8
Source File: ScalaShellRemoteStreamEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Executes the remote job.
 *
 * @param streamGraph
 *            Stream Graph to execute
 * @param jarFiles
 * 			  List of jar file URLs to ship to the cluster
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException {
	URL jarUrl;
	try {
		jarUrl = flinkILoop.writeFilesToDisk().getAbsoluteFile().toURI().toURL();
	} catch (MalformedURLException e) {
		throw new ProgramInvocationException("Could not write the user code classes to disk.",
			streamGraph.getJobGraph().getJobID(), e);
	}

	List<URL> allJarFiles = new ArrayList<>(jarFiles.size() + 1);
	allJarFiles.addAll(jarFiles);
	allJarFiles.add(jarUrl);

	return super.executeRemotely(streamGraph, allJarFiles);
}
 
Example #9
Source File: CheckpointExceptionHandlerConfigurationTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public void doTestPropagationFromCheckpointConfig(boolean failTaskOnCheckpointErrors) throws Exception {
	StreamExecutionEnvironment streamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment();
	streamExecutionEnvironment.setParallelism(1);
	streamExecutionEnvironment.getCheckpointConfig().setCheckpointInterval(1000);
	streamExecutionEnvironment.getCheckpointConfig().setFailOnCheckpointingErrors(failTaskOnCheckpointErrors);
	streamExecutionEnvironment.addSource(new SourceFunction<Integer>() {

		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
		}

		@Override
		public void cancel() {
		}

	}).addSink(new DiscardingSink<>());

	StreamGraph streamGraph = streamExecutionEnvironment.getStreamGraph();
	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);
	SerializedValue<ExecutionConfig> serializedExecutionConfig = jobGraph.getSerializedExecutionConfig();
	ExecutionConfig executionConfig =
		serializedExecutionConfig.deserializeValue(Thread.currentThread().getContextClassLoader());

	Assert.assertEquals(failTaskOnCheckpointErrors, executionConfig.isFailTaskOnCheckpointError());
}
 
Example #10
Source File: LocalStreamEnvironmentWithAsyncExecution.java    From flink-crawler with Apache License 2.0 6 votes vote down vote up
/**
 * This method lets you start a job and immediately return.
 * 
 * @param jobName
 * @return
 * @throws Exception
 */
public JobSubmissionResult executeAsync(String jobName) throws Exception {
    // transform the streaming program into a JobGraph
    StreamGraph streamGraph = getStreamGraph();
    streamGraph.setJobName(jobName);

    JobGraph jobGraph = streamGraph.getJobGraph();

    Configuration configuration = new Configuration();
    configuration.addAll(jobGraph.getJobConfiguration());

    configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS,
            jobGraph.getMaximumParallelism());

    // add (and override) the settings with what the user defined
    configuration.addAll(_conf);

    _exec = new LocalFlinkMiniCluster(configuration, true);
    _exec.start(true);

    // The above code is all basically the same as Flink's LocalStreamEnvironment.
    // The change is that here we call submitJobDetached vs. submitJobAndWait.
    // We assume that eventually someone calls stop(job id), which then terminates
    // the LocalFlinkMinimCluster.
    return _exec.submitJobDetached(jobGraph);
}
 
Example #11
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
 
Example #12
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 
Example #13
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 
Example #14
Source File: StreamContextEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	final JobClient jobClient = executeAsync(streamGraph);
	final List<JobListener> jobListeners = getJobListeners();

	try {
		final JobExecutionResult  jobExecutionResult = getJobExecutionResult(jobClient);
		jobListeners.forEach(jobListener ->
				jobListener.onJobExecuted(jobExecutionResult, null));
		return jobExecutionResult;
	} catch (Throwable t) {
		jobListeners.forEach(jobListener ->
				jobListener.onJobExecuted(null, ExceptionUtils.stripExecutionException(t)));
		ExceptionUtils.rethrowException(t);

		// never reached, only make javac happy
		return null;
	}
}
 
Example #15
Source File: IterateITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
 
Example #16
Source File: ScalaShellRemoteStreamEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Executes the remote job.
 *
 * @param streamGraph
 *            Stream Graph to execute
 * @param jarFiles
 * 			  List of jar file URLs to ship to the cluster
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
protected JobExecutionResult executeRemotely(StreamGraph streamGraph, List<URL> jarFiles) throws ProgramInvocationException {
	URL jarUrl;
	try {
		jarUrl = flinkILoop.writeFilesToDisk().getAbsoluteFile().toURI().toURL();
	} catch (MalformedURLException e) {
		throw new ProgramInvocationException("Could not write the user code classes to disk.",
			streamGraph.getJobGraph().getJobID(), e);
	}

	List<URL> allJarFiles = new ArrayList<>(jarFiles.size() + 1);
	allJarFiles.addAll(jarFiles);
	allJarFiles.add(jarUrl);

	return super.executeRemotely(streamGraph, allJarFiles);
}
 
Example #17
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that in a streaming use case where checkpointing is enabled, there is no default strategy set on the
 * client side.
 */
@Test
public void testFallbackStrategyOnClientSideWhenCheckpointingEnabled() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FallbackRestartStrategyConfiguration);
}
 
Example #18
Source File: StreamContextEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	transformations.clear();

	// execute the programs
	if (ctx instanceof DetachedEnvironment) {
		LOG.warn("Job was executed in detached mode, the results will be available on completion.");
		((DetachedEnvironment) ctx).setDetachedPlan(streamGraph);
		return DetachedEnvironment.DetachedJobExecutionResult.INSTANCE;
	} else {
		return ctx
			.getClient()
			.run(streamGraph, ctx.getJars(), ctx.getClasspaths(), ctx.getUserCodeClassLoader(), ctx.getSavepointRestoreSettings())
			.getJobExecutionResult();
	}
}
 
Example #19
Source File: RemoteStreamEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Executes the job remotely.
 *
 * <p>This method can be used independent of the {@link StreamExecutionEnvironment} type.
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@PublicEvolving
public static JobExecutionResult executeRemotely(StreamExecutionEnvironment streamExecutionEnvironment,
	List<URL> jarFiles,
	String host,
	int port,
	Configuration clientConfiguration,
	List<URL> globalClasspaths,
	String jobName,
	SavepointRestoreSettings savepointRestoreSettings
) throws ProgramInvocationException {
	StreamGraph streamGraph = streamExecutionEnvironment.getStreamGraph(jobName);
	return executeRemotely(streamGraph,
		streamExecutionEnvironment.getClass().getClassLoader(),
		streamExecutionEnvironment.getConfig(),
		jarFiles,
		host,
		port,
		clientConfiguration,
		globalClasspaths,
		savepointRestoreSettings);
}
 
Example #20
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
 
Example #21
Source File: FlinkStreamSqlEngine.java    From sylph with Apache License 2.0 5 votes vote down vote up
private static JobGraph compile(
        String jobId,
        ConnectorStore connectorStore,
        FlinkJobConfig jobConfig,
        String[] sqlSplit,
        URLClassLoader jobClassLoader)
        throws Exception
{
    JVMLauncher<JobGraph> launcher = JVMLaunchers.<JobGraph>newJvm()
            .setConsole((line) -> logger.info(new Ansi().fg(YELLOW).a("[" + jobId + "] ").fg(GREEN).a(line).reset().toString()))
            .setCallable(() -> {
                System.out.println("************ job start ***************");
                StreamExecutionEnvironment execEnv = FlinkEnvFactory.getStreamEnv(jobConfig, jobId);
                StreamTableEnvironment tableEnv = StreamTableEnvironment.create(execEnv);
                StreamSqlBuilder streamSqlBuilder = new StreamSqlBuilder(tableEnv, connectorStore, new AntlrSqlParser());
                Arrays.stream(sqlSplit).forEach(streamSqlBuilder::buildStreamBySql);
                StreamGraph streamGraph = execEnv.getStreamGraph();
                streamGraph.setJobName(jobId);
                return streamGraph.getJobGraph();
            })
            .addUserURLClassLoader(jobClassLoader)
            .setClassLoader(jobClassLoader)
            .build();

    JobGraph jobGraph = launcher.startAndGet();
    //setJobConfig(jobGraph, jobConfig, jobClassLoader, jobId);
    return jobGraph;
}
 
Example #22
Source File: BlockingPartitionBenchmark.java    From flink-benchmarks with Apache License 2.0 5 votes vote down vote up
private void executeBenchmark(StreamExecutionEnvironment env) throws Exception {
	DataStreamSource<Long> source = env.addSource(new LongSource(RECORDS_PER_INVOCATION));
	source.addSink(new DiscardingSink<>());

	StreamGraph streamGraph = env.getStreamGraph();
	streamGraph.setChaining(false);
	streamGraph.setGlobalDataExchangeMode(GlobalDataExchangeMode.ALL_EDGES_BLOCKING);
	streamGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST);

	env.execute(streamGraph);
}
 
Example #23
Source File: StreamPlanEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public JobClient executeAsync(StreamGraph streamGraph) {
	pipeline = streamGraph;

	// do not go on with anything now!
	throw new ProgramAbortException();
}
 
Example #24
Source File: StreamContextEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
	validateAllowedExecution();
	final JobClient jobClient = super.executeAsync(streamGraph);

	if (!suppressSysout) {
		System.out.println("Job has been submitted with JobID " + jobClient.getJobID());
	}

	return jobClient;
}
 
Example #25
Source File: FlinkTestUtil.java    From AthenaX with Apache License 2.0 5 votes vote down vote up
static LocalFlinkMiniCluster execute(LocalStreamEnvironment env,
                                     Configuration conf, String jobName) throws Exception {
  StreamGraph streamGraph = env.getStreamGraph();
  streamGraph.setJobName(jobName);
  JobGraph jobGraph = streamGraph.getJobGraph();
  Configuration configuration = new Configuration(conf);
  configuration.addAll(jobGraph.getJobConfiguration());
  configuration.setLong("taskmanager.memory.size", -1L);
  configuration.setInteger("taskmanager.numberOfTaskSlots", jobGraph.getMaximumParallelism());

  LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(configuration, true);
  cluster.start();
  cluster.submitJobDetached(jobGraph);
  return cluster;
}
 
Example #26
Source File: StreamExecutionEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Triggers the program execution asynchronously. The environment will execute all parts of
 * the program that have resulted in a "sink" operation. Sink operations are
 * for example printing results or forwarding them to a message queue.
 *
 * @param streamGraph the stream graph representing the transformations
 * @return A {@link JobClient} that can be used to communicate with the submitted job, completed on submission succeeded.
 * @throws Exception which occurs during job execution.
 */
@Internal
public JobClient executeAsync(StreamGraph streamGraph) throws Exception {
	checkNotNull(streamGraph, "StreamGraph cannot be null.");
	checkNotNull(configuration.get(DeploymentOptions.TARGET), "No execution.target specified in your configuration file.");

	final PipelineExecutorFactory executorFactory =
		executorServiceLoader.getExecutorFactory(configuration);

	checkNotNull(
		executorFactory,
		"Cannot find compatible factory for specified execution.target (=%s)",
		configuration.get(DeploymentOptions.TARGET));

	CompletableFuture<JobClient> jobClientFuture = executorFactory
		.getExecutor(configuration)
		.execute(streamGraph, configuration);

	try {
		JobClient jobClient = jobClientFuture.get();
		jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(jobClient, null));
		return jobClient;
	} catch (ExecutionException executionException) {
		final Throwable strippedException = ExceptionUtils.stripExecutionException(executionException);
		jobListeners.forEach(jobListener -> jobListener.onJobSubmitted(null, strippedException));

		throw new FlinkException(
			String.format("Failed to execute job '%s'.", streamGraph.getJobName()),
			strippedException);
	}
}
 
Example #27
Source File: BatchExecutor.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public Pipeline createPipeline(List<Transformation<?>> transformations, TableConfig tableConfig, String jobName) {
	StreamExecutionEnvironment execEnv = getExecutionEnvironment();
	ExecutorUtils.setBatchProperties(execEnv, tableConfig);
	StreamGraph streamGraph = ExecutorUtils.generateStreamGraph(execEnv, transformations);
	streamGraph.setJobName(getNonEmptyJobName(jobName));
	ExecutorUtils.setBatchProperties(streamGraph, tableConfig);
	return streamGraph;
}
 
Example #28
Source File: TestStreamEnvironment.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public JobExecutionResult execute(StreamGraph streamGraph) throws Exception {
	final JobGraph jobGraph = streamGraph.getJobGraph();

	for (Path jarFile : jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	return jobExecutor.executeJobBlocking(jobGraph);
}
 
Example #29
Source File: ExecutionContext.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private FlinkPlan createPlan(String name, Configuration flinkConfig) {
	if (streamExecEnv != null) {
		final StreamGraph graph = streamExecEnv.getStreamGraph();
		graph.setJobName(name);
		return graph;
	} else {
		final int parallelism = execEnv.getParallelism();
		final Plan unoptimizedPlan = execEnv.createProgramPlan();
		unoptimizedPlan.setJobName(name);
		final Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), flinkConfig);
		return ClusterClient.getOptimizedPlan(compiler, unoptimizedPlan, parallelism);
	}
}
 
Example #30
Source File: ExecutorUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Sets batch properties for {@link StreamGraph}.
 */
public static void setBatchProperties(StreamGraph streamGraph, TableConfig tableConfig) {
	streamGraph.getStreamNodes().forEach(
			sn -> sn.setResources(ResourceSpec.UNKNOWN, ResourceSpec.UNKNOWN));
	streamGraph.setChaining(true);
	streamGraph.setAllVerticesInSameSlotSharingGroupByDefault(false);
	streamGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST);
	streamGraph.setStateBackend(null);
	if (streamGraph.getCheckpointConfig().isCheckpointingEnabled()) {
		throw new IllegalArgumentException("Checkpoint is not supported for batch jobs.");
	}
	streamGraph.setGlobalDataExchangeMode(getGlobalDataExchangeMode(tableConfig));
}