org.apache.flink.optimizer.dag.OptimizerNode Java Examples

@Override
public boolean preVisit(OptimizerNode node) {

	// if the current node is a union
	if (node instanceof BinaryUnionNode) {
		int parallelism = -1;
		// set ship strategy of all outgoing connections to FORWARD.
		for (DagConnection conn : node.getOutgoingConnections()) {
			parallelism = conn.getTarget().getParallelism();
			conn.setShipStrategy(ShipStrategyType.FORWARD);
		}
		// adjust parallelism to be same as successor
		node.setParallelism(parallelism);
	}

	// traverse the whole plan
	return true;
}
 

public SolutionSetPlanNode(SolutionSetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

@Override
public boolean preVisit(OptimizerNode node) {

	// if the current node is a union
	if (node instanceof BinaryUnionNode) {
		int parallelism = -1;
		// set ship strategy of all outgoing connections to FORWARD.
		for (DagConnection conn : node.getOutgoingConnections()) {
			parallelism = conn.getTarget().getParallelism();
			conn.setShipStrategy(ShipStrategyType.FORWARD);
		}
		// adjust parallelism to be same as successor
		node.setParallelism(parallelism);
	}

	// traverse the whole plan
	return true;
}
 

public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy,
		FieldList driverKeyFields1, FieldList driverKeyFields2, boolean[] driverSortOrders)
{
	super(template, nodeName, diverStrategy);
	this.input1 = input1;
	this.input2 = input2;
	this.keys1 = driverKeyFields1;
	this.keys2 = driverKeyFields2;
	this.sortOrders = driverSortOrders;
	
	if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input1.setReplicationFactor(getParallelism());
	}
	if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input2.setReplicationFactor(getParallelism());
	}
	
	mergeBranchPlanMaps(input1.getSource(), input2.getSource());
}
 

public WorksetPlanNode(WorksetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

private void mergeBranchPlanMaps() {
	for (OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()) {
		OptimizerNode brancher = desc.getBranchingNode();

		if (branchPlan == null) {
			branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
		}
		
		if (!branchPlan.containsKey(brancher)) {
			PlanNode selectedCandidate = null;

			if (rootOfStepFunction.branchPlan != null) {
				selectedCandidate = rootOfStepFunction.branchPlan.get(brancher);
			}

			if (selectedCandidate == null) {
				throw new CompilerException(
						"Candidates for a node with open branches are missing information about the selected candidate ");
			}

			this.branchPlan.put(brancher, selectedCandidate);
		}
	}
}
 

public BulkPartialSolutionPlanNode(BulkPartialSolutionNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the partial solution does not cost anything
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

@Override
public boolean preVisit(OptimizerNode node) {

	// if the current node is a union
	if (node instanceof BinaryUnionNode) {
		int parallelism = -1;
		// set ship strategy of all outgoing connections to FORWARD.
		for (DagConnection conn : node.getOutgoingConnections()) {
			parallelism = conn.getTarget().getParallelism();
			conn.setShipStrategy(ShipStrategyType.FORWARD);
		}
		// adjust parallelism to be same as successor
		node.setParallelism(parallelism);
	}

	// traverse the whole plan
	return true;
}
 

public WorksetPlanNode(WorksetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

public SolutionSetPlanNode(SolutionSetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

public PlanNode(OptimizerNode template, String nodeName, DriverStrategy strategy) {
	this.outChannels = new ArrayList<Channel>(2);
	this.broadcastInputs = new ArrayList<NamedChannel>();
	this.template = template;
	this.nodeName = nodeName;
	this.driverStrategy = strategy;
	
	this.parallelism = template.getParallelism();

	// check, if there is branch at this node. if yes, this candidate must be associated with
	// the branching template node.
	if (template.isBranching()) {
		this.branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
		this.branchPlan.put(template, this);
	}
}
 

private void mergeBranchPlanMaps() {
	for (OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()) {
		OptimizerNode brancher = desc.getBranchingNode();

		if (branchPlan == null) {
			branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
		}
		
		if (!branchPlan.containsKey(brancher)) {
			PlanNode selectedCandidate = null;

			if (rootOfStepFunction.branchPlan != null) {
				selectedCandidate = rootOfStepFunction.branchPlan.get(brancher);
			}

			if (selectedCandidate == null) {
				throw new CompilerException(
						"Candidates for a node with open branches are missing information about the selected candidate ");
			}

			this.branchPlan.put(brancher, selectedCandidate);
		}
	}
}
 

public WorksetPlanNode(WorksetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

public SolutionSetPlanNode(SolutionSetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

public DualInputPlanNode(OptimizerNode template, String nodeName, Channel input1, Channel input2, DriverStrategy diverStrategy,
		FieldList driverKeyFields1, FieldList driverKeyFields2, boolean[] driverSortOrders)
{
	super(template, nodeName, diverStrategy);
	this.input1 = input1;
	this.input2 = input2;
	this.keys1 = driverKeyFields1;
	this.keys2 = driverKeyFields2;
	this.sortOrders = driverSortOrders;
	
	if (this.input1.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input1.setReplicationFactor(getParallelism());
	}
	if (this.input2.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input2.setReplicationFactor(getParallelism());
	}
	
	mergeBranchPlanMaps(input1.getSource(), input2.getSource());
}
 

private void mergeBranchPlanMaps() {
	for (OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()) {
		OptimizerNode brancher = desc.getBranchingNode();

		if (branchPlan == null) {
			branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
		}
		
		if (!branchPlan.containsKey(brancher)) {
			PlanNode selectedCandidate = null;

			if (rootOfStepFunction.branchPlan != null) {
				selectedCandidate = rootOfStepFunction.branchPlan.get(brancher);
			}

			if (selectedCandidate == null) {
				throw new CompilerException(
						"Candidates for a node with open branches are missing information about the selected candidate ");
			}

			this.branchPlan.put(brancher, selectedCandidate);
		}
	}
}
 

public BulkPartialSolutionPlanNode(BulkPartialSolutionNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the partial solution does not cost anything
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

public PlanNode(OptimizerNode template, String nodeName, DriverStrategy strategy) {
	this.outChannels = new ArrayList<Channel>(2);
	this.broadcastInputs = new ArrayList<NamedChannel>();
	this.template = template;
	this.nodeName = nodeName;
	this.driverStrategy = strategy;
	
	this.parallelism = template.getParallelism();

	// check, if there is branch at this node. if yes, this candidate must be associated with
	// the branching template node.
	if (template.isBranching()) {
		this.branchPlan = new HashMap<OptimizerNode, PlanNode>(6);
		this.branchPlan.put(template, this);
	}
}
 

public BulkPartialSolutionPlanNode(BulkPartialSolutionNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the partial solution does not cost anything
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}
 

@Override
public void postVisit(OptimizerNode visitable) {
	visitable.identifyDynamicPath(this.costWeight);

	// check that there is no nested iteration on the dynamic path
	if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) {
		throw new CompilerException("Nested iterations are currently not supported.");
	}
}
 

@Override
public boolean preVisit(OptimizerNode node) {
	// The interesting properties must be computed on the descend. In case a node has multiple outputs,
	// that computation must happen during the last descend.

	if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
		node.computeUnionOfInterestingPropertiesFromSuccessors();
		node.computeInterestingPropertiesForInputs(this.estimator);
		return true;
	} else {
		return false;
	}
}
 

private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism,
								ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
	if (closure == null){
		con2node = new HashMap<Operator<?>, OptimizerNode>();
	} else {
		con2node = closure;
	}

	this.sinks = new ArrayList<DataSinkNode>(2);
	this.defaultParallelism = defaultParallelism;
	this.parent = parent;
	this.defaultDataExchangeMode = dataExchangeMode;
	this.forceParallelism = forceParallelism;
}
 

@Override
public void postVisit(OptimizerNode node) {
	if (node instanceof IterationNode) {
		((IterationNode) node).acceptForStepFunction(this);
	}

	node.computeUnclosedBranchStack();
}
 

public SingleInputPlanNode(OptimizerNode template, String nodeName, Channel input, 
		DriverStrategy driverStrategy, FieldList driverKeyFields, boolean[] driverSortOrders)
{
	super(template, nodeName, driverStrategy);
	this.input = input;
	
	this.comparators = new TypeComparatorFactory<?>[driverStrategy.getNumRequiredComparators()];
	this.driverKeys = new FieldList[driverStrategy.getNumRequiredComparators()];
	this.driverSortOrders = new boolean[driverStrategy.getNumRequiredComparators()][];
	
	if(driverStrategy.getNumRequiredComparators() > 0) {
		this.driverKeys[0] = driverKeyFields;
		this.driverSortOrders[0] = driverSortOrders;
	}
	
	if (this.input.getShipStrategy() == ShipStrategyType.BROADCAST) {
		this.input.setReplicationFactor(getParallelism());
	}
	
	final PlanNode predNode = input.getSource();
	
	if (predNode.branchPlan != null && !predNode.branchPlan.isEmpty()) {
		
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		this.branchPlan.putAll(predNode.branchPlan);
	}
}
 

@Override
public void postVisit(OptimizerNode node) {
	if (node instanceof IterationNode) {
		((IterationNode) node).acceptForStepFunction(this);
	}

	node.computeUnclosedBranchStack();
}
 

@Override
public void postVisit(OptimizerNode node) {
	// if required, recurse into the step function
	if (node instanceof IterationNode) {
		((IterationNode) node).acceptForStepFunction(this);
	}
}
 

/**
 * Sets a list of all broadcast inputs attached to this node.
 */
public void setBroadcastInputs(List<NamedChannel> broadcastInputs) {
	if (broadcastInputs != null) {
		this.broadcastInputs = broadcastInputs;
		
		// update the branch map
		for (NamedChannel nc : broadcastInputs) {
			PlanNode source = nc.getSource();
			
			mergeBranchPlanMaps(branchPlan, source.branchPlan);
		}
	}
	
	// do a sanity check that if we are branching, we have now candidates for each branch point
	if (this.template.hasUnclosedBranches()) {
		if (this.branchPlan == null) {
			throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
		}

		for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
			OptimizerNode brancher = uc.getBranchingNode();
			if (this.branchPlan.get(brancher) == null) {
				throw new CompilerException("Branching and rejoining logic did not find a candidate for the branching point.");
			}
		}
	}
}
 

public PlanNode getCandidateAtBranchPoint(OptimizerNode branchPoint) {
	if (branchPlan == null) {
		return null;
	} else {
		return this.branchPlan.get(branchPoint);
	}
}
 

protected void mergeBranchPlanMaps(Map<OptimizerNode, PlanNode> branchPlan1, Map<OptimizerNode, PlanNode> branchPlan2) {
	// merge the branchPlan maps according the template's uncloseBranchesStack
	if (this.template.hasUnclosedBranches()) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>(8);
		}

		for (UnclosedBranchDescriptor uc : this.template.getOpenBranches()) {
			OptimizerNode brancher = uc.getBranchingNode();
			PlanNode selectedCandidate = null;

			if (branchPlan1 != null) {
				// predecessor 1 has branching children, see if it got the branch we are looking for
				selectedCandidate = branchPlan1.get(brancher);
			}
			
			if (selectedCandidate == null && branchPlan2 != null) {
				// predecessor 2 has branching children, see if it got the branch we are looking for
				selectedCandidate = branchPlan2.get(brancher);
			}
			
			// it may be that the branch candidate is only found once the broadcast variables are set
			if (selectedCandidate != null) {
				this.branchPlan.put(brancher, selectedCandidate);
			}
		}
	}
}
 

@Override
public boolean preVisit(OptimizerNode node) {
	// The interesting properties must be computed on the descend. In case a node has multiple outputs,
	// that computation must happen during the last descend.

	if (node.getInterestingProperties() == null && node.haveAllOutputConnectionInterestingProperties()) {
		node.computeUnionOfInterestingPropertiesFromSuccessors();
		node.computeInterestingPropertiesForInputs(this.estimator);
		return true;
	} else {
		return false;
	}
}