Java Code Examples for org.apache.flink.api.common.ExecutionConfig#PARALLELISM_DEFAULT

@Test
public void testConfigurationOfParallelism() {
	Operator operator = new MockOperator();

	// verify explicit change in parallelism
	int parallelism = 36;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());

	// verify that parallelism is reset to default flag value
	parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());
}
 

/**
 * Transforms a {@code SourceTransformation}.
 */
private <T> Collection<Integer> transformSource(SourceTransformation<T> source) {
	String slotSharingGroup = determineSlotSharingGroup(source.getSlotSharingGroup(), Collections.emptyList());

	streamGraph.addSource(source.getId(),
			slotSharingGroup,
			source.getCoLocationGroupKey(),
			source.getOperatorFactory(),
			null,
			source.getOutputType(),
			"Source: " + source.getName());
	int parallelism = source.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
			source.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(source.getId(), parallelism);
	streamGraph.setMaxParallelism(source.getId(), source.getMaxParallelism());
	return Collections.singleton(source.getId());
}
 

@Test
public void testConfigurationOfParallelism() {
	Operator operator = new MockOperator();

	// verify explicit change in parallelism
	int parallelism = 36;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());

	// verify that parallelism is reset to default flag value
	parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());
}
 

@Test
public void testConfigurationOfParallelism() {
	Operator operator = new MockOperator();

	// verify explicit change in parallelism
	int parallelism = 36;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());

	// verify that parallelism is reset to default flag value
	parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());
}
 

public void applyToConfiguration(Configuration configuration) {
	if (getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT) {
		configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, getParallelism());
	}

	configuration.setBoolean(DeploymentOptions.ATTACHED, !getDetachedMode());
	configuration.setBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED, isShutdownOnAttachedExit());
	ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, getClasspaths(), URL::toString);
	SavepointRestoreSettings.toConfiguration(getSavepointRestoreSettings(), configuration);
}
 

/**
 * Transforms a {@code OneInputTransformation}.
 *
 * <p>This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and
 * wired the inputs to this new node.
 */
private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransformation<IN, OUT> transform) {

	Collection<Integer> inputIds = transform(transform.getInput());

	// the recursive call might have already transformed this
	if (alreadyTransformed.containsKey(transform)) {
		return alreadyTransformed.get(transform);
	}

	String slotSharingGroup = determineSlotSharingGroup(transform.getSlotSharingGroup(), inputIds);

	streamGraph.addOperator(transform.getId(),
			slotSharingGroup,
			transform.getCoLocationGroupKey(),
			transform.getOperatorFactory(),
			transform.getInputType(),
			transform.getOutputType(),
			transform.getName());

	if (transform.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = transform.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(transform.getId(), transform.getStateKeySelector(), keySerializer);
	}

	int parallelism = transform.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		transform.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(transform.getId(), parallelism);
	streamGraph.setMaxParallelism(transform.getId(), transform.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId, transform.getId(), 0);
	}

	return Collections.singleton(transform.getId());
}
 

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 

private static <T> int getParallelism(MapPartitionOperator<T, TaggedOperatorSubtaskState> subtaskStates) {
	int parallelism = subtaskStates.getParallelism();
	if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) {
		parallelism = subtaskStates.getExecutionEnvironment().getParallelism();
	}

	return parallelism;
}
 

/**
 * Transforms a {@code OneInputTransformation}.
 *
 * <p>This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and
 * wired the inputs to this new node.
 */
private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransformation<IN, OUT> transform) {

	Collection<Integer> inputIds = transform(transform.getInput());

	// the recursive call might have already transformed this
	if (alreadyTransformed.containsKey(transform)) {
		return alreadyTransformed.get(transform);
	}

	String slotSharingGroup = determineSlotSharingGroup(transform.getSlotSharingGroup(), inputIds);

	streamGraph.addOperator(transform.getId(),
			slotSharingGroup,
			transform.getCoLocationGroupKey(),
			transform.getOperatorFactory(),
			transform.getInputType(),
			transform.getOutputType(),
			transform.getName());

	if (transform.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = transform.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(transform.getId(), transform.getStateKeySelector(), keySerializer);
	}

	int parallelism = transform.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		transform.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(transform.getId(), parallelism);
	streamGraph.setMaxParallelism(transform.getId(), transform.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId, transform.getId(), 0);
	}

	return Collections.singleton(transform.getId());
}
 

/**
 * Transforms a {@code SinkTransformation}.
 */
private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) {

	Collection<Integer> inputIds = transform(sink.getInput());

	String slotSharingGroup = determineSlotSharingGroup(sink.getSlotSharingGroup(), inputIds);

	streamGraph.addSink(sink.getId(),
			slotSharingGroup,
			sink.getCoLocationGroupKey(),
			sink.getOperatorFactory(),
			sink.getInput().getOutputType(),
			null,
			"Sink: " + sink.getName());

	StreamOperatorFactory operatorFactory = sink.getOperatorFactory();
	if (operatorFactory instanceof OutputFormatOperatorFactory) {
		streamGraph.setOutputFormat(sink.getId(), ((OutputFormatOperatorFactory) operatorFactory).getOutputFormat());
	}

	int parallelism = sink.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		sink.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(sink.getId(), parallelism);
	streamGraph.setMaxParallelism(sink.getId(), sink.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId,
				sink.getId(),
				0
		);
	}

	if (sink.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = sink.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer);
	}

	return Collections.emptyList();
}
 

/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws Exception {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	final ProgramOptions programOptions = ProgramOptions.create(commandLine);

	// evaluate help flag
	if (commandLine.hasOption(HELP_OPTION.getOpt())) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(programOptions);

	try {
		int parallelism = programOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		final CustomCommandLine activeCommandLine =
				validateAndGetActiveCommandLine(checkNotNull(commandLine));

		final Configuration effectiveConfiguration = getEffectiveConfiguration(
				activeCommandLine, commandLine, programOptions, program.getJobJarAndDependencies());

		Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, effectiveConfiguration, parallelism, true);
		String jsonPlan = FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 

/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws CliArgsException, FileNotFoundException, ProgramInvocationException {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	InfoOptions infoOptions = new InfoOptions(commandLine);

	// evaluate help flag
	if (infoOptions.isPrintHelp()) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	if (infoOptions.getJarFilePath() == null) {
		throw new CliArgsException("The program JAR file was not specified.");
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(infoOptions);

	try {
		int parallelism = infoOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
		FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);

		String jsonPlan = null;
		if (flinkPlan instanceof OptimizedPlan) {
			jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
		} else if (flinkPlan instanceof StreamingPlan) {
			jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
		}

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 

@Override
public String toString() {
	return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism())
			+ ") : " + getIdString();
}
 

/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws CliArgsException, FileNotFoundException, ProgramInvocationException {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	InfoOptions infoOptions = new InfoOptions(commandLine);

	// evaluate help flag
	if (infoOptions.isPrintHelp()) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	if (infoOptions.getJarFilePath() == null) {
		throw new CliArgsException("The program JAR file was not specified.");
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(infoOptions);

	try {
		int parallelism = infoOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
		FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);

		String jsonPlan = null;
		if (flinkPlan instanceof OptimizedPlan) {
			jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
		} else if (flinkPlan instanceof StreamingPlan) {
			jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
		}

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 

@Override
public String toString() {
	return "Local Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism()) + ").";
}
 

private static int resolveParallelism(int parallelism, Configuration configuration) {
  return (parallelism == ExecutionConfig.PARALLELISM_DEFAULT)
      ? configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM)
      : parallelism;
}
 

@Override
public String toString() {
	return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism())
			+ ") : " + getIdString();
}
 

/**
 * Sets the parallelism for this optimizer node.
 * The parallelism denotes how many parallel instances of the operator will be
 * spawned during the execution.
 * 
 * @param parallelism The parallelism to set. If this value is {@link ExecutionConfig#PARALLELISM_DEFAULT}
 *        then the system will take the default number of parallel instances.
 * @throws IllegalArgumentException If the parallelism is smaller than one.
 */
public void setParallelism(int parallelism) {
	if (parallelism < 1 && parallelism != ExecutionConfig.PARALLELISM_DEFAULT) {
		throw new IllegalArgumentException("Parallelism of " + parallelism + " is invalid.");
	}
	this.parallelism = parallelism;
}
 

/**
 * Sets the parallelism for this optimizer node.
 * The parallelism denotes how many parallel instances of the operator will be
 * spawned during the execution.
 * 
 * @param parallelism The parallelism to set. If this value is {@link ExecutionConfig#PARALLELISM_DEFAULT}
 *        then the system will take the default number of parallel instances.
 * @throws IllegalArgumentException If the parallelism is smaller than one.
 */
public void setParallelism(int parallelism) {
	if (parallelism < 1 && parallelism != ExecutionConfig.PARALLELISM_DEFAULT) {
		throw new IllegalArgumentException("Parallelism of " + parallelism + " is invalid.");
	}
	this.parallelism = parallelism;
}