Java Code Examples for org.apache.flink.optimizer.plan.PlanNode

The following examples show how to use org.apache.flink.optimizer.plan.PlanNode. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify operator parallelism.
 *
 * @param env the Flink execution environment.
 * @param expectedParallelism expected operator parallelism
 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
	env.setParallelism(2 * expectedParallelism);

	Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());

	List<PlanNode> queue = new ArrayList<>();
	queue.addAll(optimizedPlan.getDataSinks());

	while (queue.size() > 0) {
		PlanNode node = queue.remove(queue.size() - 1);

		// Data sources may have parallelism of 1, so simply check that the node
		// parallelism has not been increased by setting the default parallelism
		assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);

		for (Channel channel : node.getInputs()) {
			queue.add(channel.getSource());
		}
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: SingleInputNode.java    License: Apache License 2.0 6 votes vote down vote up
protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 
Example 3
Source Project: flink   Source File: OptimizerNode.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks whether to candidate plans for the sub-plan of this node are comparable. The two
 * alternative plans are comparable, if
 * 
 * a) There is no branch in the sub-plan of this node
 * b) Both candidates have the same candidate as the child at the last open branch. 
 * 
 * @param plan1 The root node of the first candidate plan.
 * @param plan2 The root node of the second candidate plan.
 * @return True if the nodes are branch compatible in the inputs.
 */
protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
	if (plan1 == null || plan2 == null) {
		throw new NullPointerException();
	}
	
	// if there is no open branch, the children are always compatible.
	// in most plans, that will be the dominant case
	if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
		return true;
	}

	for (OptimizerNode joinedBrancher : hereJoinedBranches) {
		final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
		final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
		
		if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
			return false;
		}
	}
	return true;
}
 
Example 4
Source Project: Flink-CEPplus   Source File: CompilerTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name and stub class.");
	} else {
		PlanNode found = null;
		for (PlanNode node : nodes) {
			if (node.getClass() == stubClass) {
				if (found == null) {
					found = node;
				} else {
					throw new RuntimeException("Multiple nodes found with the given name and stub class.");
				}
			}
		}
		if (found == null) {
			throw new RuntimeException("No node found with the given name and stub class.");
		} else {
			return (T) found;
		}
	}
}
 
Example 5
Source Project: flink   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify operator parallelism.
 *
 * @param env the Flink execution environment.
 * @param expectedParallelism expected operator parallelism
 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
	env.setParallelism(2 * expectedParallelism);

	Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());

	List<PlanNode> queue = new ArrayList<>();
	queue.addAll(optimizedPlan.getDataSinks());

	while (queue.size() > 0) {
		PlanNode node = queue.remove(queue.size() - 1);

		// Data sources may have parallelism of 1, so simply check that the node
		// parallelism has not been increased by setting the default parallelism
		assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);

		for (Channel channel : node.getInputs()) {
			queue.add(channel.getSource());
		}
	}
}
 
Example 6
Source Project: flink   Source File: SingleInputNode.java    License: Apache License 2.0 6 votes vote down vote up
protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 
Example 7
Source Project: flink   Source File: OptimizerNode.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks whether to candidate plans for the sub-plan of this node are comparable. The two
 * alternative plans are comparable, if
 * 
 * a) There is no branch in the sub-plan of this node
 * b) Both candidates have the same candidate as the child at the last open branch. 
 * 
 * @param plan1 The root node of the first candidate plan.
 * @param plan2 The root node of the second candidate plan.
 * @return True if the nodes are branch compatible in the inputs.
 */
protected boolean areBranchCompatible(PlanNode plan1, PlanNode plan2) {
	if (plan1 == null || plan2 == null) {
		throw new NullPointerException();
	}
	
	// if there is no open branch, the children are always compatible.
	// in most plans, that will be the dominant case
	if (this.hereJoinedBranches == null || this.hereJoinedBranches.isEmpty()) {
		return true;
	}

	for (OptimizerNode joinedBrancher : hereJoinedBranches) {
		final PlanNode branch1Cand = plan1.getCandidateAtBranchPoint(joinedBrancher);
		final PlanNode branch2Cand = plan2.getCandidateAtBranchPoint(joinedBrancher);
		
		if (branch1Cand != null && branch2Cand != null && branch1Cand != branch2Cand) {
			return false;
		}
	}
	return true;
}
 
Example 8
Source Project: flink   Source File: SingleInputNode.java    License: Apache License 2.0 6 votes vote down vote up
protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}
 
Example 9
Source Project: flink   Source File: CompilerTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public <T extends PlanNode> T getNode(String name, Class<? extends Function> stubClass) {
	List<PlanNode> nodes = this.map.get(name);
	if (nodes == null || nodes.isEmpty()) {
		throw new RuntimeException("No node found with the given name and stub class.");
	} else {
		PlanNode found = null;
		for (PlanNode node : nodes) {
			if (node.getClass() == stubClass) {
				if (found == null) {
					found = node;
				} else {
					throw new RuntimeException("Multiple nodes found with the given name and stub class.");
				}
			}
		}
		if (found == null) {
			throw new RuntimeException("No node found with the given name and stub class.");
		} else {
			return (T) found;
		}
	}
}
 
Example 10
Source Project: flink   Source File: TestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify operator parallelism.
 *
 * @param env the Flink execution environment.
 * @param expectedParallelism expected operator parallelism
 */
public static void verifyParallelism(ExecutionEnvironment env, int expectedParallelism) {
	env.setParallelism(2 * expectedParallelism);

	Optimizer compiler = new Optimizer(null, new DefaultCostEstimator(), new Configuration());
	OptimizedPlan optimizedPlan = compiler.compile(env.createProgramPlan());

	List<PlanNode> queue = new ArrayList<>();
	queue.addAll(optimizedPlan.getDataSinks());

	while (queue.size() > 0) {
		PlanNode node = queue.remove(queue.size() - 1);

		// Data sources may have parallelism of 1, so simply check that the node
		// parallelism has not been increased by setting the default parallelism
		assertTrue("Wrong parallelism for " + node.toString(), node.getParallelism() <= expectedParallelism);

		for (Channel channel : node.getInputs()) {
			queue.add(channel.getSource());
		}
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private void assignDriverResources(PlanNode node, TaskConfig config) {
	final double relativeMem = node.getRelativeMemoryPerSubTask();
	if (relativeMem > 0) {
		config.setRelativeMemoryDriver(relativeMem);
		config.setFilehandlesDriver(this.defaultMaxFan);
		config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
TaskInChain(PlanNode planNode, Class<? extends ChainedDriver<?, ?>> chainedTask,
			TaskConfig taskConfig, String taskName) {
	
	this.planNode = planNode;
	this.chainedTask = chainedTask;
	this.taskConfig = taskConfig;
	this.taskName = taskName;
}
 
Example 13
Source Project: Flink-CEPplus   Source File: WorksetNode.java    License: Apache License 2.0 5 votes vote down vote up
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
		this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
	}
}
 
Example 14
Source Project: flink   Source File: BulkPartialSolutionNode.java    License: Apache License 2.0 5 votes vote down vote up
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
				"PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: AbstractPartialSolutionNode.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
	if (this.cachedPlans != null) {
		return this.cachedPlans;
	} else {
		throw new IllegalStateException();
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: BulkPartialSolutionNode.java    License: Apache License 2.0 5 votes vote down vote up
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
				"PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) {
	PlanNode source = channel.getSource();
	Operator<?> javaOp = source.getProgramOperator();
	TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
	return createComparator(type, channel.getShipStrategyKeys(),
		getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig);
}
 
Example 18
Source Project: Flink-CEPplus   Source File: RangePartitionRewriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postVisit(PlanNode node) {

	if(node instanceof IterationPlanNode) {
		IterationPlanNode iNode = (IterationPlanNode)node;
		if(!visitedIterationNodes.contains(iNode)) {
			visitedIterationNodes.add(iNode);
			iNode.acceptForStepFunction(this);
		}
	}

	final Iterable<Channel> inputChannels = node.getInputs();
	for (Channel channel : inputChannels) {
		ShipStrategyType shipStrategy = channel.getShipStrategy();
		// Make sure we only optimize the DAG for range partition, and do not optimize multi times.
		if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {

			if(channel.getDataDistribution() == null) {
				if (node.isOnDynamicPath()) {
					throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
				}

				PlanNode channelSource = channel.getSource();
				List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
				channelSource.getOutgoingChannels().remove(channel);
				channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
			}
		}
	}
}
 
Example 19
Source Project: Flink-CEPplus   Source File: PlanFinalizer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new plan finalizer.
 */
public PlanFinalizer() {
	this.allNodes = new HashSet<PlanNode>();
	this.sources = new ArrayList<SourcePlanNode>();
	this.sinks = new ArrayList<SinkPlanNode>();
	this.stackOfIterationNodes = new ArrayDeque<IterationPlanNode>();
}
 
Example 20
Source Project: flink   Source File: BinaryUnionReplacer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean preVisit(PlanNode visitable) {
	if (this.seenBefore.add(visitable)) {
		if (visitable instanceof IterationPlanNode) {
			((IterationPlanNode) visitable).acceptForStepFunction(this);
		}
		return true;
	} else {
		return false;
	}
}
 
Example 21
Source Project: flink   Source File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private void assignDriverResources(PlanNode node, TaskConfig config) {
	final double relativeMem = node.getRelativeMemoryPerSubTask();
	if (relativeMem > 0) {
		config.setRelativeMemoryDriver(relativeMem);
		config.setFilehandlesDriver(this.defaultMaxFan);
		config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
	}
}
 
Example 22
Source Project: flink   Source File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private boolean checkAndConfigurePersistentIntermediateResult(PlanNode node) {
	if (!(node instanceof SinkPlanNode)) {
		return false;
	}

	final Object userCodeObject = node.getProgramOperator().getUserCodeWrapper().getUserCodeObject();
	if (!(userCodeObject instanceof BlockingShuffleOutputFormat)) {
		return false;
	}

	final Iterator<Channel> inputIterator = node.getInputs().iterator();
	checkState(inputIterator.hasNext(), "SinkPlanNode must have a input.");

	final PlanNode predecessorNode = inputIterator.next().getSource();
	final JobVertex predecessorVertex = (vertices.containsKey(predecessorNode)) ?
		vertices.get(predecessorNode) :
		chainedTasks.get(predecessorNode).getContainingVertex();

	checkState(predecessorVertex != null, "Bug: Chained task has not been assigned its containing vertex when connecting.");

	predecessorVertex.createAndAddResultDataSet(
			// use specified intermediateDataSetID
			new IntermediateDataSetID(((BlockingShuffleOutputFormat) userCodeObject).getIntermediateDataSetId()),
			ResultPartitionType.BLOCKING_PERSISTENT);

	// remove this node so the OutputFormatVertex will not shown in the final JobGraph.
	vertices.remove(node);
	return true;
}
 
Example 23
Source Project: flink   Source File: JobGraphGenerator.java    License: Apache License 2.0 5 votes vote down vote up
TaskInChain(PlanNode planNode, Class<? extends ChainedDriver<?, ?>> chainedTask,
			TaskConfig taskConfig, String taskName) {
	
	this.planNode = planNode;
	this.chainedTask = chainedTask;
	this.taskConfig = taskConfig;
	this.taskName = taskName;
}
 
Example 24
Source Project: flink   Source File: AbstractPartialSolutionNode.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
	if (this.cachedPlans != null) {
		return this.cachedPlans;
	} else {
		throw new IllegalStateException();
	}
}
 
Example 25
Source Project: flink   Source File: WorksetNode.java    License: Apache License 2.0 5 votes vote down vote up
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
		this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
	}
}
 
Example 26
Source Project: flink   Source File: AbstractPartialSolutionNode.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
	if (this.cachedPlans != null) {
		return this.cachedPlans;
	} else {
		throw new IllegalStateException();
	}
}
 
Example 27
Source Project: flink   Source File: RangePartitionRewriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postVisit(PlanNode node) {

	if(node instanceof IterationPlanNode) {
		IterationPlanNode iNode = (IterationPlanNode)node;
		if(!visitedIterationNodes.contains(iNode)) {
			visitedIterationNodes.add(iNode);
			iNode.acceptForStepFunction(this);
		}
	}

	final Iterable<Channel> inputChannels = node.getInputs();
	for (Channel channel : inputChannels) {
		ShipStrategyType shipStrategy = channel.getShipStrategy();
		// Make sure we only optimize the DAG for range partition, and do not optimize multi times.
		if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {

			if(channel.getDataDistribution() == null) {
				if (node.isOnDynamicPath()) {
					throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
				}

				PlanNode channelSource = channel.getSource();
				List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
				channelSource.getOutgoingChannels().remove(channel);
				channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
			}
		}
	}
}
 
Example 28
Source Project: flink   Source File: BulkPartialSolutionNode.java    License: Apache License 2.0 5 votes vote down vote up
public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		this.cachedPlans = Collections.<PlanNode>singletonList(new BulkPartialSolutionPlanNode(this,
				"PartialSolution ("+this.getOperator().getName()+")", gProps, lProps, initialInput));
	}
}
 
Example 29
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static TypeComparatorFactory<?> getShipComparator(Channel channel, ExecutionConfig executionConfig) {
	PlanNode source = channel.getSource();
	Operator<?> javaOp = source.getProgramOperator();
	TypeInformation<?> type = javaOp.getOperatorInfo().getOutputType();
	return createComparator(type, channel.getShipStrategyKeys(),
		getSortOrders(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()), executionConfig);
}
 
Example 30
Source Project: flink   Source File: RangePartitionRewriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void postVisit(PlanNode node) {

	if(node instanceof IterationPlanNode) {
		IterationPlanNode iNode = (IterationPlanNode)node;
		if(!visitedIterationNodes.contains(iNode)) {
			visitedIterationNodes.add(iNode);
			iNode.acceptForStepFunction(this);
		}
	}

	final Iterable<Channel> inputChannels = node.getInputs();
	for (Channel channel : inputChannels) {
		ShipStrategyType shipStrategy = channel.getShipStrategy();
		// Make sure we only optimize the DAG for range partition, and do not optimize multi times.
		if (shipStrategy == ShipStrategyType.PARTITION_RANGE) {

			if(channel.getDataDistribution() == null) {
				if (node.isOnDynamicPath()) {
					throw new InvalidProgramException("Range Partitioning not supported within iterations if users do not supply the data distribution.");
				}

				PlanNode channelSource = channel.getSource();
				List<Channel> newSourceOutputChannels = rewriteRangePartitionChannel(channel);
				channelSource.getOutgoingChannels().remove(channel);
				channelSource.getOutgoingChannels().addAll(newSourceOutputChannels);
			}
		}
	}
}