Java Code Examples for org.apache.flink.api.common.ExecutionConfig#PARALLELISM_DEFAULT

The following examples show how to use org.apache.flink.api.common.ExecutionConfig#PARALLELISM_DEFAULT . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: StreamGraphGenerator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Transforms a {@code SourceTransformation}.
 */
private <T> Collection<Integer> transformSource(SourceTransformation<T> source) {
	String slotSharingGroup = determineSlotSharingGroup(source.getSlotSharingGroup(), Collections.emptyList());

	streamGraph.addSource(source.getId(),
			slotSharingGroup,
			source.getCoLocationGroupKey(),
			source.getOperatorFactory(),
			null,
			source.getOutputType(),
			"Source: " + source.getName());
	int parallelism = source.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
			source.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(source.getId(), parallelism);
	streamGraph.setMaxParallelism(source.getId(), source.getMaxParallelism());
	return Collections.singleton(source.getId());
}
 
Example 2
Source Project: Flink-CEPplus   File: OperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigurationOfParallelism() {
	Operator operator = new MockOperator();

	// verify explicit change in parallelism
	int parallelism = 36;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());

	// verify that parallelism is reset to default flag value
	parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());
}
 
Example 3
Source Project: flink   File: OperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigurationOfParallelism() {
	Operator operator = new MockOperator();

	// verify explicit change in parallelism
	int parallelism = 36;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());

	// verify that parallelism is reset to default flag value
	parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());
}
 
Example 4
Source Project: flink   File: OperatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigurationOfParallelism() {
	Operator operator = new MockOperator();

	// verify explicit change in parallelism
	int parallelism = 36;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());

	// verify that parallelism is reset to default flag value
	parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
	operator.setParallelism(parallelism);

	assertEquals(parallelism, operator.getParallelism());
}
 
Example 5
Source Project: flink   File: StreamGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Transforms a {@code SinkTransformation}.
 */
private <T> Collection<Integer> transformSink(SinkTransformation<T> sink) {

	Collection<Integer> inputIds = transform(sink.getInput());

	String slotSharingGroup = determineSlotSharingGroup(sink.getSlotSharingGroup(), inputIds);

	streamGraph.addSink(sink.getId(),
			slotSharingGroup,
			sink.getCoLocationGroupKey(),
			sink.getOperatorFactory(),
			sink.getInput().getOutputType(),
			null,
			"Sink: " + sink.getName());

	StreamOperatorFactory operatorFactory = sink.getOperatorFactory();
	if (operatorFactory instanceof OutputFormatOperatorFactory) {
		streamGraph.setOutputFormat(sink.getId(), ((OutputFormatOperatorFactory) operatorFactory).getOutputFormat());
	}

	int parallelism = sink.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		sink.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(sink.getId(), parallelism);
	streamGraph.setMaxParallelism(sink.getId(), sink.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId,
				sink.getId(),
				0
		);
	}

	if (sink.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = sink.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(sink.getId(), sink.getStateKeySelector(), keySerializer);
	}

	return Collections.emptyList();
}
 
Example 6
Source Project: flink   File: StreamGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Transforms a {@code OneInputTransformation}.
 *
 * <p>This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and
 * wired the inputs to this new node.
 */
private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransformation<IN, OUT> transform) {

	Collection<Integer> inputIds = transform(transform.getInput());

	// the recursive call might have already transformed this
	if (alreadyTransformed.containsKey(transform)) {
		return alreadyTransformed.get(transform);
	}

	String slotSharingGroup = determineSlotSharingGroup(transform.getSlotSharingGroup(), inputIds);

	streamGraph.addOperator(transform.getId(),
			slotSharingGroup,
			transform.getCoLocationGroupKey(),
			transform.getOperatorFactory(),
			transform.getInputType(),
			transform.getOutputType(),
			transform.getName());

	if (transform.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = transform.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(transform.getId(), transform.getStateKeySelector(), keySerializer);
	}

	int parallelism = transform.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		transform.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(transform.getId(), parallelism);
	streamGraph.setMaxParallelism(transform.getId(), transform.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId, transform.getId(), 0);
	}

	return Collections.singleton(transform.getId());
}
 
Example 7
Source Project: flink   File: BootstrapTransformation.java    License: Apache License 2.0 5 votes vote down vote up
private static <T> int getParallelism(MapPartitionOperator<T, TaggedOperatorSubtaskState> subtaskStates) {
	int parallelism = subtaskStates.getParallelism();
	if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) {
		parallelism = subtaskStates.getExecutionEnvironment().getParallelism();
	}

	return parallelism;
}
 
Example 8
Source Project: flink   File: LocalExecutor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
@Override
public String getOptimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), this.baseConfiguration);
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 
Example 9
Source Project: flink   File: LocalExecutor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a JSON representation of the given dataflow's execution plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow's execution plan, as a JSON string.
 * @throws Exception Thrown, if the optimization process that creates the execution plan failed.
 */
public static String optimizerPlanAsJSON(Plan plan) throws Exception {
	final int parallelism = plan.getDefaultParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? 1 : plan.getDefaultParallelism();

	Optimizer pc = new Optimizer(new DataStatistics(), new Configuration());
	pc.setDefaultParallelism(parallelism);
	OptimizedPlan op = pc.compile(plan);

	return new PlanJSONDumpGenerator().getOptimizerPlanAsJSON(op);
}
 
Example 10
Source Project: flink   File: StreamGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Transforms a {@code OneInputTransformation}.
 *
 * <p>This recursively transforms the inputs, creates a new {@code StreamNode} in the graph and
 * wired the inputs to this new node.
 */
private <IN, OUT> Collection<Integer> transformOneInputTransform(OneInputTransformation<IN, OUT> transform) {

	Collection<Integer> inputIds = transform(transform.getInput());

	// the recursive call might have already transformed this
	if (alreadyTransformed.containsKey(transform)) {
		return alreadyTransformed.get(transform);
	}

	String slotSharingGroup = determineSlotSharingGroup(transform.getSlotSharingGroup(), inputIds);

	streamGraph.addOperator(transform.getId(),
			slotSharingGroup,
			transform.getCoLocationGroupKey(),
			transform.getOperatorFactory(),
			transform.getInputType(),
			transform.getOutputType(),
			transform.getName());

	if (transform.getStateKeySelector() != null) {
		TypeSerializer<?> keySerializer = transform.getStateKeyType().createSerializer(executionConfig);
		streamGraph.setOneInputStateKey(transform.getId(), transform.getStateKeySelector(), keySerializer);
	}

	int parallelism = transform.getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT ?
		transform.getParallelism() : executionConfig.getParallelism();
	streamGraph.setParallelism(transform.getId(), parallelism);
	streamGraph.setMaxParallelism(transform.getId(), transform.getMaxParallelism());

	for (Integer inputId: inputIds) {
		streamGraph.addEdge(inputId, transform.getId(), 0);
	}

	return Collections.singleton(transform.getId());
}
 
Example 11
Source Project: flink   File: ProgramOptions.java    License: Apache License 2.0 5 votes vote down vote up
public void applyToConfiguration(Configuration configuration) {
	if (getParallelism() != ExecutionConfig.PARALLELISM_DEFAULT) {
		configuration.setInteger(CoreOptions.DEFAULT_PARALLELISM, getParallelism());
	}

	configuration.setBoolean(DeploymentOptions.ATTACHED, !getDetachedMode());
	configuration.setBoolean(DeploymentOptions.SHUTDOWN_IF_ATTACHED, isShutdownOnAttachedExit());
	ConfigUtils.encodeCollectionToConfig(configuration, PipelineOptions.CLASSPATHS, getClasspaths(), URL::toString);
	SavepointRestoreSettings.toConfiguration(getSavepointRestoreSettings(), configuration);
}
 
Example 12
@Override
public String toString() {
	return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism())
			+ ") : " + getIdString();
}
 
Example 13
Source Project: Flink-CEPplus   File: CliFrontend.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws CliArgsException, FileNotFoundException, ProgramInvocationException {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	InfoOptions infoOptions = new InfoOptions(commandLine);

	// evaluate help flag
	if (infoOptions.isPrintHelp()) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	if (infoOptions.getJarFilePath() == null) {
		throw new CliArgsException("The program JAR file was not specified.");
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(infoOptions);

	try {
		int parallelism = infoOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
		FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);

		String jsonPlan = null;
		if (flinkPlan instanceof OptimizedPlan) {
			jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
		} else if (flinkPlan instanceof StreamingPlan) {
			jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
		}

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 
Example 14
Source Project: flink   File: CliFrontend.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws Exception {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	final ProgramOptions programOptions = ProgramOptions.create(commandLine);

	// evaluate help flag
	if (commandLine.hasOption(HELP_OPTION.getOpt())) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(programOptions);

	try {
		int parallelism = programOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		final CustomCommandLine activeCommandLine =
				validateAndGetActiveCommandLine(checkNotNull(commandLine));

		final Configuration effectiveConfiguration = getEffectiveConfiguration(
				activeCommandLine, commandLine, programOptions, program.getJobJarAndDependencies());

		Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, effectiveConfiguration, parallelism, true);
		String jsonPlan = FlinkPipelineTranslationUtil.translateToJSONExecutionPlan(pipeline);

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 
Example 15
Source Project: flink   File: ContextEnvironment.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String toString() {
	return "Context Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism())
			+ ") : " + getIdString();
}
 
Example 16
Source Project: flink   File: CliFrontend.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Executes the info action.
 *
 * @param args Command line arguments for the info action.
 */
protected void info(String[] args) throws CliArgsException, FileNotFoundException, ProgramInvocationException {
	LOG.info("Running 'info' command.");

	final Options commandOptions = CliFrontendParser.getInfoCommandOptions();

	final CommandLine commandLine = CliFrontendParser.parse(commandOptions, args, true);

	InfoOptions infoOptions = new InfoOptions(commandLine);

	// evaluate help flag
	if (infoOptions.isPrintHelp()) {
		CliFrontendParser.printHelpForInfo();
		return;
	}

	if (infoOptions.getJarFilePath() == null) {
		throw new CliArgsException("The program JAR file was not specified.");
	}

	// -------- build the packaged program -------------

	LOG.info("Building program from JAR file");
	final PackagedProgram program = buildProgram(infoOptions);

	try {
		int parallelism = infoOptions.getParallelism();
		if (ExecutionConfig.PARALLELISM_DEFAULT == parallelism) {
			parallelism = defaultParallelism;
		}

		LOG.info("Creating program plan dump");

		Optimizer compiler = new Optimizer(new DataStatistics(), new DefaultCostEstimator(), configuration);
		FlinkPlan flinkPlan = ClusterClient.getOptimizedPlan(compiler, program, parallelism);

		String jsonPlan = null;
		if (flinkPlan instanceof OptimizedPlan) {
			jsonPlan = new PlanJSONDumpGenerator().getOptimizerPlanAsJSON((OptimizedPlan) flinkPlan);
		} else if (flinkPlan instanceof StreamingPlan) {
			jsonPlan = ((StreamingPlan) flinkPlan).getStreamingPlanAsJSON();
		}

		if (jsonPlan != null) {
			System.out.println("----------------------- Execution Plan -----------------------");
			System.out.println(jsonPlan);
			System.out.println("--------------------------------------------------------------");
		}
		else {
			System.out.println("JSON plan could not be generated.");
		}

		String description = program.getDescription();
		if (description != null) {
			System.out.println();
			System.out.println(description);
		}
		else {
			System.out.println();
			System.out.println("No description provided.");
		}
	}
	finally {
		program.deleteExtractedLibraries();
	}
}
 
Example 17
Source Project: flink   File: LocalEnvironment.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public String toString() {
	return "Local Environment (parallelism = " + (getParallelism() == ExecutionConfig.PARALLELISM_DEFAULT ? "default" : getParallelism()) + ").";
}
 
Example 18
private static int resolveParallelism(int parallelism, Configuration configuration) {
  return (parallelism == ExecutionConfig.PARALLELISM_DEFAULT)
      ? configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM)
      : parallelism;
}
 
Example 19
Source Project: Flink-CEPplus   File: OptimizerNode.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Sets the parallelism for this optimizer node.
 * The parallelism denotes how many parallel instances of the operator will be
 * spawned during the execution.
 * 
 * @param parallelism The parallelism to set. If this value is {@link ExecutionConfig#PARALLELISM_DEFAULT}
 *        then the system will take the default number of parallel instances.
 * @throws IllegalArgumentException If the parallelism is smaller than one.
 */
public void setParallelism(int parallelism) {
	if (parallelism < 1 && parallelism != ExecutionConfig.PARALLELISM_DEFAULT) {
		throw new IllegalArgumentException("Parallelism of " + parallelism + " is invalid.");
	}
	this.parallelism = parallelism;
}
 
Example 20
Source Project: flink   File: OptimizerNode.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Sets the parallelism for this optimizer node.
 * The parallelism denotes how many parallel instances of the operator will be
 * spawned during the execution.
 * 
 * @param parallelism The parallelism to set. If this value is {@link ExecutionConfig#PARALLELISM_DEFAULT}
 *        then the system will take the default number of parallel instances.
 * @throws IllegalArgumentException If the parallelism is smaller than one.
 */
public void setParallelism(int parallelism) {
	if (parallelism < 1 && parallelism != ExecutionConfig.PARALLELISM_DEFAULT) {
		throw new IllegalArgumentException("Parallelism of " + parallelism + " is invalid.");
	}
	this.parallelism = parallelism;
}