org.apache.flink.client.program.PackagedProgramUtils Java Examples

The following examples show how to use org.apache.flink.client.program.PackagedProgramUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ClassPathJobGraphRetriever.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
	final PackagedProgram packagedProgram = createPackagedProgram();
	final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
	try {
		final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(
			packagedProgram,
			configuration,
			defaultParallelism,
			jobId);
		jobGraph.setAllowQueuedScheduling(true);
		jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

		return jobGraph;
	} catch (Exception e) {
		throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
	}
}
 
Example #2
Source File: JarHandlerUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public JobGraph toJobGraph(Configuration configuration) {
	if (!Files.exists(jarFile)) {
		throw new CompletionException(new RestHandlerException(
			String.format("Jar file %s does not exist", jarFile), HttpResponseStatus.BAD_REQUEST));
	}

	try {
		final PackagedProgram packagedProgram = new PackagedProgram(
			jarFile.toFile(),
			entryClass,
			programArgs.toArray(new String[0]));
		return PackagedProgramUtils.createJobGraph(packagedProgram, configuration, parallelism, jobId);
	} catch (final ProgramInvocationException e) {
		throw new CompletionException(e);
	}
}
 
Example #3
Source File: ClassPathJobGraphRetriever.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
	final PackagedProgram packagedProgram = createPackagedProgram();
	final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
	try {
		final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(
			packagedProgram,
			configuration,
			defaultParallelism,
			jobId);
		jobGraph.setAllowQueuedScheduling(true);
		jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

		return jobGraph;
	} catch (Exception e) {
		throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
	}
}
 
Example #4
Source File: JarHandlerUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public JobGraph toJobGraph(Configuration configuration) {
	if (!Files.exists(jarFile)) {
		throw new CompletionException(new RestHandlerException(
			String.format("Jar file %s does not exist", jarFile), HttpResponseStatus.BAD_REQUEST));
	}

	try {
		final PackagedProgram packagedProgram = new PackagedProgram(
			jarFile.toFile(),
			entryClass,
			programArgs.toArray(new String[0]));
		return PackagedProgramUtils.createJobGraph(packagedProgram, configuration, parallelism, jobId);
	} catch (final ProgramInvocationException e) {
		throw new CompletionException(e);
	}
}
 
Example #5
Source File: StatefulFunctionsJobGraphRetriever.java    From stateful-functions with Apache License 2.0 6 votes vote down vote up
@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
  final PackagedProgram packagedProgram = createPackagedProgram();

  final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
  try {
    final JobGraph jobGraph =
        PackagedProgramUtils.createJobGraph(
            packagedProgram, configuration, defaultParallelism, jobId, false);
    jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

    return jobGraph;
  } catch (Exception e) {
    throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
  }
}
 
Example #6
Source File: StatefulFunctionsJobGraphRetriever.java    From flink-statefun with Apache License 2.0 6 votes vote down vote up
@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
  final PackagedProgram packagedProgram = createPackagedProgram();

  int resolvedParallelism = resolveParallelism(parallelism, configuration);
  LOG.info(
      "Creating JobGraph for job {}, with parallelism {} and savepoint restore settings {}.",
      jobId,
      resolvedParallelism,
      savepointRestoreSettings);
  try {
    final JobGraph jobGraph =
        PackagedProgramUtils.createJobGraph(
            packagedProgram, configuration, resolvedParallelism, jobId, false);
    jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

    return jobGraph;
  } catch (Exception e) {
    throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
  }
}
 
Example #7
Source File: PreviewPlanDumpTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void verifyPlanDump(Class<?> entrypoint, String... args) throws Exception {
	final PackagedProgram program = PackagedProgram
		.newBuilder()
		.setEntryPointClassName(entrypoint.getName())
		.setArguments(args)
		.build();

	final Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, new Configuration(), 1, true);

	assertTrue(pipeline instanceof Plan);

	final Plan plan = (Plan) pipeline;

	final List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
	final PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
	final String json = dumper.getPactPlanAsJSON(sinks);

	try (JsonParser parser = new JsonFactory().createParser(json)) {
		while (parser.nextToken() != null) {
		}
	}
}
 
Example #8
Source File: DumpCompiledPlanTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void verifyOptimizedPlan(Class<?> entrypoint, String... args) throws Exception {
	final PackagedProgram program = PackagedProgram
		.newBuilder()
		.setEntryPointClassName(entrypoint.getName())
		.setArguments(args)
		.build();

	final Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, new Configuration(), 1, true);

	assertTrue(pipeline instanceof Plan);

	final Plan plan = (Plan) pipeline;

	final OptimizedPlan op = compileNoStats(plan);
	final PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
	final String json = dumper.getOptimizerPlanAsJSON(op);
	try (JsonParser parser = new JsonFactory().createParser(json)) {
		while (parser.nextToken() != null) {
		}
	}
}
 
Example #9
Source File: KubernetesUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static List<File> checkJarFileForApplicationMode(Configuration configuration) {
	return configuration.get(PipelineOptions.JARS).stream().map(
		FunctionUtils.uncheckedFunction(
			uri -> {
				final URI jarURI = PackagedProgramUtils.resolveURI(uri);
				if (jarURI.getScheme().equals("local") && jarURI.isAbsolute()) {
					return new File(jarURI.getPath());
				}
				throw new IllegalArgumentException("Only \"local\" is supported as schema for application mode." +
						" This assumes that the jar is located in the image, not the Flink client." +
						" An example of such path is: local:///opt/flink/examples/streaming/WindowJoin.jar");
			})
	).collect(Collectors.toList());
}
 
Example #10
Source File: JarHandlerUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public JobGraph toJobGraph(Configuration configuration, boolean suppressOutput) {
	try {
		final PackagedProgram packagedProgram = toPackagedProgram(configuration);
		return PackagedProgramUtils.createJobGraph(packagedProgram, configuration, parallelism, jobId, suppressOutput);
	} catch (final ProgramInvocationException e) {
		throw new CompletionException(e);
	}
}
 
Example #11
Source File: CliFrontend.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void runApplication(String[] args) throws Exception {
	LOG.info("Running 'run-application' command.");

	final Options commandOptions = CliFrontendParser.getRunCommandOptions();
	final CommandLine commandLine = getCommandLine(commandOptions, args, true);

	if (commandLine.hasOption(HELP_OPTION.getOpt())) {
		CliFrontendParser.printHelpForRun(customCommandLines);
		return;
	}

	final CustomCommandLine activeCommandLine =
			validateAndGetActiveCommandLine(checkNotNull(commandLine));

	final ProgramOptions programOptions = new ProgramOptions(commandLine);

	final ApplicationDeployer deployer =
			new ApplicationClusterDeployer(clusterClientServiceLoader);

	programOptions.validate();
	final URI uri = PackagedProgramUtils.resolveURI(programOptions.getJarFilePath());
	final Configuration effectiveConfiguration = getEffectiveConfiguration(
			activeCommandLine, commandLine, programOptions, Collections.singletonList(uri.toString()));
	final ApplicationConfiguration applicationConfiguration =
			new ApplicationConfiguration(programOptions.getProgramArgs(), programOptions.getEntryPointClassName());
	deployer.run(effectiveConfiguration, applicationConfiguration);
}
 
Example #12
Source File: ClassPathPackagedProgramRetrieverTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private JobGraph retrieveJobGraph(ClassPathPackagedProgramRetriever retrieverUnderTest, Configuration configuration) throws FlinkException, ProgramInvocationException, MalformedURLException {
	final PackagedProgram packagedProgram = retrieverUnderTest.getPackagedProgram();

	final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
	ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.JARS, packagedProgram.getJobJarAndDependencies(), URL::toString);
	ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, packagedProgram.getClasspaths(), URL::toString);

	final Pipeline pipeline = PackagedProgramUtils
			.getPipelineFromProgram(packagedProgram, configuration, defaultParallelism, false);
	return PipelineExecutorUtils.getJobGraph(pipeline, configuration);
}
 
Example #13
Source File: CliFrontend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws Exception {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	final ProgramOptions programOptions = ProgramOptions.create(commandLine);

	// evaluate help flag
	if (commandLine.hasOption(HELP_OPTION.getOpt())) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(programOptions);

	try {
		int parallelism = programOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		final CustomCommandLine activeCommandLine =
				validateAndGetActiveCommandLine(checkNotNull(commandLine));

		final Configuration effectiveConfiguration = getEffectiveConfiguration(
				activeCommandLine, commandLine, programOptions, program.getJobJarAndDependencies());

		Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, effectiveConfiguration, parallelism, true);
		String jsonPlan = FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 
Example #14
Source File: CliFrontendPackageProgramTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Ensure that we will never have the following error.
 *
 * <pre>
 * 	org.apache.flink.client.program.ProgramInvocationException: The main method caused an error.
 *		at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:398)
 *		at org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:301)
 *		at org.apache.flink.client.program.Client.getOptimizedPlan(Client.java:140)
 *		at org.apache.flink.client.program.Client.getOptimizedPlanAsJson(Client.java:125)
 *		at org.apache.flink.client.cli.CliFrontend.info(CliFrontend.java:439)
 *		at org.apache.flink.client.cli.CliFrontend.parseParameters(CliFrontend.java:931)
 *		at org.apache.flink.client.cli.CliFrontend.main(CliFrontend.java:951)
 *	Caused by: java.io.IOException: java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hadoop.hive.ql.io.RCFileInputFormat
 *		at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:102)
 *		at org.apache.hcatalog.mapreduce.HCatInputFormat.setInput(HCatInputFormat.java:54)
 *		at tlabs.CDR_In_Report.createHCatInputFormat(CDR_In_Report.java:322)
 *		at tlabs.CDR_Out_Report.main(CDR_Out_Report.java:380)
 *		at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
 *		at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
 *		at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
 *		at java.lang.reflect.Method.invoke(Method.java:622)
 *		at org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:383)
 * </pre>
 *
 * <p>The test works as follows:
 *
 * <ul>
 *   <li> Use the CliFrontend to invoke a jar file that loads a class which is only available
 * 	      in the jarfile itself (via a custom classloader)
 *   <li> Change the Usercode classloader of the PackagedProgram to a special classloader for this test
 *   <li> the classloader will accept the special class (and return a String.class)
 * </ul>
 */
@Test
public void testPlanWithExternalClass() throws Exception {
	final boolean[] callme = { false }; // create a final object reference, to be able to change its val later

	try {
		String[] arguments = {
				"--classpath", "file:///tmp/foo",
				"--classpath", "file:///tmp/bar",
				"-c", TEST_JAR_CLASSLOADERTEST_CLASS, getTestJarPath(),
				"true", "arg1", "arg2" };
		URL[] classpath = new URL[] { new URL("file:///tmp/foo"), new URL("file:///tmp/bar") };
		String[] reducedArguments = { "true", "arg1", "arg2" };

		CommandLine commandLine = CliFrontendParser.parse(CliFrontendParser.RUN_OPTIONS, arguments, true);
		ProgramOptions programOptions = ProgramOptions.create(commandLine);

		assertEquals(getTestJarPath(), programOptions.getJarFilePath());
		assertArrayEquals(classpath, programOptions.getClasspaths().toArray());
		assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, programOptions.getEntryPointClassName());
		assertArrayEquals(reducedArguments, programOptions.getProgramArgs());

		PackagedProgram prog = spy(frontend.buildProgram(programOptions));

		ClassLoader testClassLoader = new ClassLoader(prog.getUserCodeClassLoader()) {
			@Override
			public Class<?> loadClass(String name) throws ClassNotFoundException {
				if ("org.apache.hadoop.hive.ql.io.RCFileInputFormat".equals(name)) {
					callme[0] = true;
					return String.class; // Intentionally return the wrong class.
				} else {
					return super.loadClass(name);
				}
			}
		};
		when(prog.getUserCodeClassLoader()).thenReturn(testClassLoader);

		assertEquals(TEST_JAR_CLASSLOADERTEST_CLASS, prog.getMainClassName());
		assertArrayEquals(reducedArguments, prog.getArguments());

		Configuration c = new Configuration();
		Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), c);

		// we expect this to fail with a "ClassNotFoundException"
		Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(prog, c, 666, true);
		FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);
		fail("Should have failed with a ClassNotFoundException");
	}
	catch (ProgramInvocationException e) {
		if (!(e.getCause() instanceof ClassNotFoundException)) {
			e.printStackTrace();
			fail("Program didn't throw ClassNotFoundException");
		}
		assertTrue("Classloader was not called", callme[0]);
	}
}